diff --git a/build/Targets.fs b/build/Targets.fs index d9bdab6ad..88d32b137 100644 --- a/build/Targets.fs +++ b/build/Targets.fs @@ -57,6 +57,7 @@ let private pristineCheck (arguments:ParseResults) = let private publishBinaries _ = exec { run "dotnet" "publish" "src/docs-builder/docs-builder.csproj" } exec { run "dotnet" "publish" "src/docs-generator/docs-generator.csproj" } + exec { run "dotnet" "publish" "src/docs-assembler/docs-assembler.csproj" } Zip.zip ".artifacts/publish/docs-builder/release" $"docs-builder-%s{OS.Name}-{OS.Arch}.zip" @@ -103,6 +104,7 @@ let private publishContainers _ = exec { run "dotnet" (args @ registry) } createImage "docs-builder" createImage "docs-generator" + createImage "docs-assembler" let private runTests _ = exec { diff --git a/docs-builder.sln b/docs-builder.sln index f5845e4cc..13e4cc936 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -48,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "publish-vercel", "publish-v EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "build", "build\build.fsproj", "{10857974-6CF1-42B5-B793-AAA988BD7348}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler", "src\docs-assembler\docs-assembler.csproj", "{28350800-B44B-479B-86E2-1D39E321C0B4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -82,6 +84,10 @@ Global {10857974-6CF1-42B5-B793-AAA988BD7348}.Debug|Any CPU.Build.0 = Debug|Any CPU {10857974-6CF1-42B5-B793-AAA988BD7348}.Release|Any CPU.ActiveCfg = Release|Any CPU {10857974-6CF1-42B5-B793-AAA988BD7348}.Release|Any CPU.Build.0 = Release|Any CPU + {28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} @@ -91,5 +97,6 @@ Global {1C340CCF-9AAC-4163-A7BB-60528076E98B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {CD2887E3-BDA9-434B-A5BF-9ED38DE20332} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {A2A34BBC-CB5E-4100-9529-A12B6ECB769C} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} + {28350800-B44B-479B-86E2-1D39E321C0B4} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} EndGlobalSection EndGlobal diff --git a/src/docs-assembler/AssemblyConfiguration.cs b/src/docs-assembler/AssemblyConfiguration.cs new file mode 100644 index 000000000..8664e79f3 --- /dev/null +++ b/src/docs-assembler/AssemblyConfiguration.cs @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using YamlDotNet.Serialization; + +namespace Documentation.Assembler; + +[YamlStaticContext] +[YamlSerializable(typeof(AssemblyConfiguration))] +[YamlSerializable(typeof(Repository))] +public partial class YamlStaticContext; + +public record AssemblyConfiguration +{ + public static AssemblyConfiguration Deserialize(string yaml) + { + var input = new StringReader(yaml); + + var deserializer = new StaticDeserializerBuilder(new YamlStaticContext()) + .IgnoreUnmatchedProperties() + .Build(); + + try + { + var config = deserializer.Deserialize(input); + return config; + } + catch (Exception e) + { + Console.WriteLine(e); + Console.WriteLine(e.InnerException); + throw; + } + } + + [YamlMember(Alias = "repos")] + public Dictionary Repositories { get; set; } = new(); +} + +public record Repository +{ + [YamlMember(Alias = "repo")] + public string Origin { get; set; } = string.Empty; + + [YamlMember(Alias = "branch")] + public string? Branch { get; set; } + +} diff --git a/src/docs-assembler/Cli/Filters.cs b/src/docs-assembler/Cli/Filters.cs new file mode 100644 index 000000000..c5ede3c44 --- /dev/null +++ b/src/docs-assembler/Cli/Filters.cs @@ -0,0 +1,51 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics; +using ConsoleAppFramework; + +namespace Documentation.Assembler.Cli; + +internal class StopwatchFilter(ConsoleAppFilter next) : ConsoleAppFilter(next) +{ + public override async Task InvokeAsync(ConsoleAppContext context, Cancel ctx) + { + var isHelpOrVersion = context.Arguments.Any(a => a is "--help" or "-h" or "--version"); + var name = string.IsNullOrWhiteSpace(context.CommandName) ? "generate" : context.CommandName; + var startTime = Stopwatch.GetTimestamp(); + if (!isHelpOrVersion) + ConsoleApp.Log($"{name} :: Starting..."); + try + { + await Next.InvokeAsync(context, ctx); + } + finally + { + var endTime = Stopwatch.GetElapsedTime(startTime); + if (!isHelpOrVersion) + ConsoleApp.Log($"{name} :: Finished in '{endTime}"); + } + } +} + +internal class CatchExceptionFilter(ConsoleAppFilter next) : ConsoleAppFilter(next) +{ + public override async Task InvokeAsync(ConsoleAppContext context, CancellationToken cancellationToken) + { + try + { + await Next.InvokeAsync(context, cancellationToken); + } + catch (Exception ex) + { + if (ex is OperationCanceledException) + { + ConsoleApp.Log("Cancellation requested, exiting."); + return; + } + + ConsoleApp.LogError(ex.ToString()); // .ToString() shows stacktrace, .Message can avoid showing stacktrace to user. + } + } +} diff --git a/src/docs-assembler/Program.cs b/src/docs-assembler/Program.cs new file mode 100644 index 000000000..cf4ff2631 --- /dev/null +++ b/src/docs-assembler/Program.cs @@ -0,0 +1,91 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Concurrent; +using System.Diagnostics; +using ConsoleAppFramework; +using Documentation.Assembler; +using Documentation.Assembler.Cli; +using Elastic.Markdown.IO; +using ProcNet; +using ProcNet.Std; + +var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml"); +var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile)); + +var app = ConsoleApp.Create(); +app.UseFilter(); +app.UseFilter(); + +// would love to use libgit2 so there is no git dependency but +// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674 +app.Add("clone-all", async Task (CancellationToken ctx) => +{ + Console.WriteLine(config.Repositories.Count); + var dict = new ConcurrentDictionary(); + await Parallel.ForEachAsync(config.Repositories, new ParallelOptions + { + CancellationToken = ctx, + MaxDegreeOfParallelism = Environment.ProcessorCount / 4 + }, async (kv, c) => + { + await Task.Run(() => + { + var name = kv.Key; + var repository = kv.Value; + var checkoutFolder = Path.Combine(Paths.Root.FullName, $".artifacts/assembly/{name}"); + + var sw = Stopwatch.StartNew(); + dict.AddOrUpdate(name, sw, (_, _) => sw); + Console.WriteLine($"Checkout: {name}\t{repository}\t{checkoutFolder}"); + var branch = repository.Branch ?? "main"; + var args = new StartArguments( + "git", "clone", repository.Origin, checkoutFolder, "--depth", "1" + , "--single-branch", "--branch", branch + ); + Proc.StartRedirected(args, new ConsoleLineHandler(name)); + sw.Stop(); + }, c); + }).ConfigureAwait(false); + + foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed)) + Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}"); +}); +app.Add("list", async Task (CancellationToken ctx) => +{ + + var assemblyPath = Path.Combine(Paths.Root.FullName, $".artifacts/assembly"); + var dir = new DirectoryInfo(assemblyPath); + var dictionary = new Dictionary(); + foreach (var d in dir.GetDirectories()) + { + var checkoutFolder = Path.Combine(assemblyPath, d.Name); + + var capture = Proc.Start( + new StartArguments("git", "rev-parse", "--abbrev-ref", "HEAD") + { + WorkingDirectory = checkoutFolder + } + ); + dictionary.Add(d.Name, capture.ConsoleOut.FirstOrDefault()?.Line ?? "unknown"); + } + foreach (var kv in dictionary.OrderBy(kv => kv.Value)) + Console.WriteLine($"-> {kv.Key}\tbranch: {kv.Value}"); + + await Task.CompletedTask; +}); + +await app.RunAsync(args); + +namespace Documentation.Assembler +{ + public class ConsoleLineHandler(string prefix) : IConsoleLineHandler + { + public void Handle(LineOut lineOut) => lineOut.CharsOrString( + r => Console.Write(prefix + ": " + r), + l => Console.WriteLine(prefix + ": " + l)); + + public void Handle(Exception e) { } + } +} diff --git a/src/docs-assembler/conf.yml b/src/docs-assembler/conf.yml new file mode 100644 index 000000000..ddf0facb5 --- /dev/null +++ b/src/docs-assembler/conf.yml @@ -0,0 +1,163 @@ +repos: + apm-aws-lambda: + repo: "git@github.com:elastic/apm-aws-lambda.git" + apm-k8s-attacher: + repo: "git@github.com:elastic/apm-k8s-attacher.git" + apm-server: + repo: "git@github.com:elastic/apm-server.git" + apm-agent-android: + repo: "git@github.com:elastic/apm-agent-android.git" + apm-agent-nodejs: + repo: "git@github.com:elastic/apm-agent-nodejs.git" + apm-agent-python: + repo: "git@github.com:elastic/apm-agent-python.git" + apm-agent-ruby: + repo: "git@github.com:elastic/apm-agent-ruby.git" + apm-agent-rum-js: + repo: "git@github.com:elastic/apm-agent-rum-js.git" + apm-agent-go: + repo: "git@github.com:elastic/apm-agent-go.git" + apm-agent-java: + repo: "git@github.com:elastic/apm-agent-java.git" + apm-agent-dotnet: + repo: "git@github.com:elastic/apm-agent-dotnet.git" + apm-agent-php: + repo: "git@github.com:elastic/apm-agent-php.git" + apm-agent-ios: + repo: "git@github.com:elastic/apm-agent-ios.git" + azure-marketplace: + repo: "git@github.com:elastic/azure-marketplace.git" + branch: master + beats: + repo: "git@github.com:elastic/beats.git" + clients-team: + repo: "git@github.com:elastic/clients-team.git" + cloud: + repo: "git@github.com:elastic/cloud.git" + branch: master + cloud-assets: + repo: "git@github.com:elastic/cloud-assets.git" + branch: master + cloud-on-k8s: + repo: "git@github.com:elastic/cloud-on-k8s.git" + curator: + repo: "git@github.com:elastic/curator.git" + branch: master + docs-content: + repo: "git@github.com:elastic/docs-content.git" + ecctl: + repo: "git@github.com:elastic/ecctl.git" + branch: master + ecs: + repo: "git@github.com:elastic/ecs.git" + ecs-dotnet: + repo: "git@github.com:elastic/ecs-dotnet.git" + ecs-logging: + repo: "git@github.com:elastic/ecs-logging.git" + ecs-logging-go-logrus: + repo: "git@github.com:elastic/ecs-logging-go-logrus.git" + ecs-logging-go-zap: + repo: "git@github.com:elastic/ecs-logging-go-zap.git" + ecs-logging-go-zerolog: + repo: "git@github.com:elastic/ecs-logging-go-zerolog.git" + ecs-logging-java: + repo: "git@github.com:elastic/ecs-logging-java.git" + ecs-logging-nodejs: + repo: "git@github.com:elastic/ecs-logging-nodejs.git" + ecs-logging-php: + repo: "git@github.com:elastic/ecs-logging-php.git" + ecs-logging-python: + repo: "git@github.com:elastic/ecs-logging-python.git" + ecs-logging-ruby: + repo: "git@github.com:elastic/ecs-logging-ruby.git" + eland: + repo: "git@github.com:elastic/eland.git" + elasticsearch-hadoop: + repo: "git@github.com:elastic/elasticsearch-hadoop.git" + elasticsearch-java: + repo: "git@github.com:elastic/elasticsearch-java.git" + elasticsearch-js: + repo: "git@github.com:elastic/elasticsearch-js.git" + elasticsearch-ruby: + repo: "git@github.com:elastic/elasticsearch-ruby.git" + elasticsearch-net: + repo: "git@github.com:elastic/elasticsearch-net.git" + elasticsearch-php: + repo: "git@github.com:elastic/elasticsearch-php.git" + elasticsearch-py: + repo: "git@github.com:elastic/elasticsearch-py.git" + elasticsearch-perl: + repo: "git@github.com:elastic/elasticsearch-perl.git" + branch: master + go-elasticsearch: + repo: "git@github.com:elastic/go-elasticsearch.git" + elasticsearch-rs: + repo: "git@github.com:elastic/elasticsearch-rs.git" + elasticsearch: + repo: "git@github.com:elastic/elasticsearch.git" + enterprise-search-pubs: + repo: "git@github.com:elastic/enterprise-search-pubs.git" + enterprise-search-js: + repo: "git@github.com:elastic/enterprise-search-js.git" + enterprise-search-php: + repo: "git@github.com:elastic/enterprise-search-php.git" + enterprise-search-python: + repo: "git@github.com:elastic/enterprise-search-python.git" + enterprise-search-ruby: + repo: "git@github.com:elastic/enterprise-search-ruby.git" + esf: + repo: "git@github.com:elastic/elastic-serverless-forwarder.git" + guide: + repo: "git@github.com:elastic/elasticsearch-definitive-guide.git" + ingest-docs: + repo: "git@github.com:elastic/ingest-docs.git" + kibana: + repo: "git@github.com:elastic/kibana.git" + logstash: + repo: "git@github.com:elastic/logstash.git" + logstash-docs: + repo: "git@github.com:elastic/logstash-docs.git" + observability-docs: + repo: "git@github.com:elastic/observability-docs.git" + package-spec: + repo: "git@github.com:elastic/package-spec.git" + security-docs: + repo: "git@github.com:elastic/security-docs.git" + sense: + repo: "git@github.com:elastic/sense.git" + branch: master + stack-docs: + repo: "git@github.com:elastic/stack-docs.git" + swiftype: + repo: "git@github.com:elastic/swiftype-doc-placeholder.git" + branch: master + tech-content: + repo: "git@github.com:elastic/tech-content.git" + terraform-provider-ec: + repo: "git@github.com:elastic/terraform-provider-ec.git" + branch: master + x-pack: + repo: "git@github.com:elastic/x-pack.git" + branch: master + x-pack-elasticsearch: + repo: "git@github.com:elastic/x-pack-elasticsearch.git" + branch: master + x-pack-kibana: + repo: "git@github.com:elastic/x-pack-kibana.git" + branch: master + x-pack-logstash: + repo: "git@github.com:elastic/x-pack-logstash.git" + + elasticsearch-js-legacy: + repo: "git@github.com:elastic/elasticsearch-js-legacy.git" + branch: 16.x + + kibana-cn: + repo: "git@github.com:elasticsearch-cn/kibana.git" + branch: cn + guide-cn: + repo: "git@github.com:elasticsearch-cn/elasticsearch-definitive-guide.git" + branch: cn + elasticsearch-php-cn: + repo: "git@github.com:elasticsearch-cn/elasticsearch-php.git" + branch: cn diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj new file mode 100644 index 000000000..0e1d83e28 --- /dev/null +++ b/src/docs-assembler/docs-assembler.csproj @@ -0,0 +1,31 @@ + + + + net9.0 + Exe + docs-assembler + Documentation.Assembler + true + + false + true + true + + true + true + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + diff --git a/src/docs-generator/docs-generator.csproj b/src/docs-generator/docs-generator.csproj index d8a35bed5..fb35054c3 100644 --- a/src/docs-generator/docs-generator.csproj +++ b/src/docs-generator/docs-generator.csproj @@ -6,16 +6,13 @@ docs-generator Documentation.Generator true + false true true true true - -