diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76cf56c8c..610166de6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,6 @@ jobs: integration: runs-on: docs-builder-latest-16 - if: false steps: - uses: actions/checkout@v4 diff --git a/aspire/AppHost.cs b/aspire/AppHost.cs new file mode 100644 index 000000000..e99f405dd --- /dev/null +++ b/aspire/AppHost.cs @@ -0,0 +1,136 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + + +using ConsoleAppFramework; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Microsoft.Extensions.Logging; +using static Elastic.Documentation.Aspire.ResourceNames; + +// ReSharper disable UnusedVariable +// ReSharper disable RedundantAssignment +// ReSharper disable NotAccessedVariable + +var logLevel = LogLevel.Information; +GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); +var globalArguments = new List(); +if (skipPrivateRepositories) + globalArguments.Add("--skip-private-repositories"); + +if (logLevel != LogLevel.Information) +{ + globalArguments.Add("--log-level"); + globalArguments.Add(logLevel.ToString()); +} + +await ConsoleApp.RunAsync(args, BuildAspireHost); +return; + +// ReSharper disable once RedundantLambdaParameterType +async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx) +{ + var builder = DistributedApplication.CreateBuilder(args); + skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories"); + + var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true); + var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true); + + var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true); + var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true); + + var cloneAll = builder.AddProject(AssemblerClone); + string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : []; + cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]); + + var buildAll = builder.AddProject(AssemblerBuild) + .WithArgs(["repo", "build-all", .. globalArguments]) + .WaitForCompletion(cloneAll) + .WithParentRelationship(cloneAll); + + var elasticsearchLocal = builder.AddElasticsearch(ElasticsearchLocal) + .WithEnvironment("LICENSE", "trial"); + if (!startElasticsearch) + elasticsearchLocal = elasticsearchLocal.WithExplicitStart(); + + var elasticsearchRemote = builder.AddExternalService(ElasticsearchRemote, elasticsearchUrl); + + var api = builder.AddProject(LambdaApi) + .WithArgs(globalArguments) + .WithEnvironment("ENVIRONMENT", "dev") + .WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl) + .WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath) + .WithExplicitStart(); + + api = startElasticsearch + ? api + .WithReference(elasticsearchLocal) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http")) + .WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter) + .WithParentRelationship(elasticsearchLocal) + .WaitFor(elasticsearchLocal) + : api.WithReference(elasticsearchRemote) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl) + .WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey); + + var indexElasticsearch = builder.AddProject(ElasticsearchIndexerPlain) + .WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments]) + .WithExplicitStart() + .WaitForCompletion(cloneAll); + indexElasticsearch = startElasticsearch + ? indexElasticsearch + .WaitFor(elasticsearchLocal) + .WithReference(elasticsearchLocal) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http")) + .WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter) + .WithParentRelationship(elasticsearchLocal) + : indexElasticsearch + .WithReference(elasticsearchRemote) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl) + .WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey) + .WithParentRelationship(elasticsearchRemote); + + var indexElasticsearchSemantic = builder.AddProject(ElasticsearchIndexerSemantic) + .WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments]) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http")) + .WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter) + .WithExplicitStart() + .WaitForCompletion(cloneAll); + indexElasticsearchSemantic = startElasticsearch + ? indexElasticsearchSemantic + .WaitFor(elasticsearchLocal) + .WithReference(elasticsearchLocal) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http")) + .WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter) + .WithParentRelationship(elasticsearchLocal) + : indexElasticsearchSemantic + .WithReference(elasticsearchRemote) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl) + .WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey) + .WithParentRelationship(elasticsearchRemote); + + var serveStatic = builder.AddProject(AssemblerServe) + .WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl) + .WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath) + .WithHttpEndpoint(port: 4000, isProxied: false) + .WithArgs(["serve-static", .. globalArguments]) + .WithHttpHealthCheck("/", 200) + .WaitForCompletion(buildAll) + .WithParentRelationship(cloneAll); + + serveStatic = startElasticsearch + ? serveStatic + .WithReference(elasticsearchLocal) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http")) + .WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter) + : serveStatic + .WithReference(elasticsearchRemote) + .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl) + .WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey); + + + serveStatic = startElasticsearch ? serveStatic.WaitFor(elasticsearchLocal) : serveStatic.WaitFor(buildAll); + + await builder.Build().RunAsync(ctx); +} diff --git a/tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json b/aspire/Properties/launchSettings.json similarity index 100% rename from tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json rename to aspire/Properties/launchSettings.json diff --git a/aspire/README.md b/aspire/README.md new file mode 100644 index 000000000..b6fb4b92a --- /dev/null +++ b/aspire/README.md @@ -0,0 +1,96 @@ +# Aspire for Elastic Documentation + +We use [Aspire](https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview) for local development purposes to spin up all services in a controlled fashion. + +> Aspire provides tools, templates, and packages for building observable, production-ready distributed apps. At the center is the app model—a code-first, single source of truth that defines your app's services, resources, and connections. +>Aspire gives you a unified toolchain: launch and debug your entire app locally with one command, then deploy anywhere—Kubernetes, the cloud, or your own servers—using the same composition. + +We do not use Aspire to generate production deployment scripts since [this is not fully baked for AWS and terraform yet](https://github.com/dotnet/aspire/issues/6559) + +![service-graph.png](service-graph.png) + +## Run all services locally + +You may need to install the Aspire workload first. We also recommend installing the aspire plugin + +* [For Rider](https://plugins.jetbrains.com/plugin/23289--net-aspire) + +```bash +sudo dotnet workload install aspire +``` + +Aspire is just another CLI program so can be run like all the other tools + +```bash +dotnet run --project aspire +``` + +This will automatically: + +* clone all repositories according to `config/assembler.yml` using `docs-assembler repo clone-all` +* do a full site build of all repositories using `docs-assembler repo build-all` +* Serve a copy of the fully assembled documentation using `docs-builder serve-static`. + +This should start a management UI over at: https://localhost:17166. This UI exposes all logs, traces, and metrics for each service + +![management-ui.png](management-ui.png) + +### Run without authorization tokens + +If you do not have access to clone to private repositories you can use `--skip-private-repositories` + +```bash +dotnet run --project aspire -- --skip-private-repositories +``` + +This will automagically scrub the private repositories from assembler.yml and navigation.yml. + +Our integration tests, for instance, use this to run tests on CI tokenless. When specifying this option locally we automatically inject `docs-builder`'s own docs into the `navigation.yml`. This allows us to test changes to documentation sets and their effect on assembler during PR's + +## Elasticsearch Instance + +By default, we assume local [dotnet user secrets](#user-secrets) have been set to communicate to an external Elasticsearch instance. + +However, you can start a local Elasticsearch instance using + +```bash +dotnet run --project aspire -- --start-elasticsearch +``` + +This will run a local Elasticsearch docker image and expose that to Aspire service discovery instead. + +### Elasticsearch indexing + +Furthermore, it makes the following indexers available in the Aspire UI + +* Plain Elasticsearch, index elasticsearch documents. +* Semantic Elasticsearch, same but with semantic fields. + +These have to be run manually and can be run multiple times. + +## User secrets + +We use dotnet user secrets to provide parameters to aspire. These are all optional but needed if you want +the AI prompts and external Elasticsearch searches to work. + +NOTE: using `--start-elasticsearch` the url and random password are automatically wired. + +```bash +dotnet user-secrets --project aspire list +``` + +Should have these secrets + +> Parameters:LlmGatewayUrl = https://**** +> Parameters:LlmGatewayServiceAccountPath = +> Parameters:DocumentationElasticUrl = https://*.elastic.cloud:443 +> Parameters:DocumentationElasticApiKey = **** + +To set them: + +```bash +dotnet user-secrets --project aspire set Parameters:DocumentationElasticApiKey +``` + +Do note `dotnet user-secrets` should only be used on local development machines and not on CI. + diff --git a/aspire/ResourceNames.cs b/aspire/ResourceNames.cs new file mode 100644 index 000000000..942899b80 --- /dev/null +++ b/aspire/ResourceNames.cs @@ -0,0 +1,17 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Documentation.Aspire; + +public static class ResourceNames +{ + public const string AssemblerClone = "assembler-clone"; + public const string AssemblerBuild = "assembler-build"; + public const string AssemblerServe = "assembler-serve"; + public const string ElasticsearchLocal = "elasticsearch-local"; + public const string ElasticsearchRemote = "elasticsearch-remote"; + public const string LambdaApi = "lambda-api"; + public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain"; + public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic"; +} diff --git a/tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json b/aspire/appsettings.Development.json similarity index 100% rename from tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json rename to aspire/appsettings.Development.json diff --git a/tests-integration/Elastic.Documentation.Aspire/appsettings.json b/aspire/appsettings.json similarity index 100% rename from tests-integration/Elastic.Documentation.Aspire/appsettings.json rename to aspire/appsettings.json diff --git a/aspire/aspire.csproj b/aspire/aspire.csproj new file mode 100644 index 000000000..07d40fb5e --- /dev/null +++ b/aspire/aspire.csproj @@ -0,0 +1,31 @@ + + + + + + Exe + net9.0 + enable + enable + 72f50f33-6fb9-4d08-bff3-39568fe370b3 + false + Elastic.Documentation.Aspire + IDE0350 + + + + + + + + + + + + + + + + + + diff --git a/aspire/management-ui.png b/aspire/management-ui.png new file mode 100644 index 000000000..46e16bc4f Binary files /dev/null and b/aspire/management-ui.png differ diff --git a/aspire/service-graph.png b/aspire/service-graph.png new file mode 100644 index 000000000..261c5936f Binary files /dev/null and b/aspire/service-graph.png differ diff --git a/docs-builder.sln b/docs-builder.sln index b9be4d877..dd6035845 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -129,7 +129,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests- EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Assembler.IntegrationTests", "tests-integration\Elastic.Assembler.IntegrationTests\Elastic.Assembler.IntegrationTests.csproj", "{A272D3EC-FAAF-4795-A796-302725382AFF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Aspire", "tests-integration\Elastic.Documentation.Aspire\Elastic.Documentation.Aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "aspire", "aspire\aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.ServiceDefaults", "src\Elastic.Documentation.ServiceDefaults\Elastic.Documentation.ServiceDefaults.csproj", "{2A83ED35-B631-4F02-8D4C-15611D0DB72C}" EndProject @@ -286,7 +286,6 @@ Global {111E7029-BB29-4039-9B45-04776798A8DD} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {164F55EC-9412-4CD4-81AD-3598B57632A6} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5} {A272D3EC-FAAF-4795-A796-302725382AFF} = {BCAD38D5-6C83-46E2-8398-4BE463931098} - {4DFECE72-4A1F-4B58-918E-DCD07B585231} = {BCAD38D5-6C83-46E2-8398-4BE463931098} {2A83ED35-B631-4F02-8D4C-15611D0DB72C} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {B042CC78-5060-4091-B95A-79C71BA3908A} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {F30B90AD-1A01-4A6F-9699-809FA6875B22} = {B042CC78-5060-4091-B95A-79C71BA3908A} diff --git a/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs b/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs index c04138cb2..655f6baf5 100644 --- a/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs +++ b/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs @@ -30,6 +30,23 @@ public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRep var repository = RepositoryDefaults(r, name); config.ReferenceRepositories[name] = repository; } + + // if we are not running in CI, and we are skipping private repositories, and we can locate the solution directory. build the local docs-content repository + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) + && skipPrivateRepositories + && config.ReferenceRepositories.TryGetValue("docs-builder", out var docsContentRepository) + && Paths.GetSolutionDirectory() is { } solutionDir + ) + { + var docsRepositoryPath = Path.Combine(solutionDir.FullName, "docs"); + config.ReferenceRepositories["docs-builder"] = docsContentRepository with + { + Skip = false, + Path = docsRepositoryPath + }; + } + + var privateRepositories = config.ReferenceRepositories.Where(r => r.Value.Private).ToList(); foreach (var (name, _) in privateRepositories) { @@ -45,7 +62,9 @@ public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRep .Where(r => !r.Skip) .Concat([config.Narrative]).ToDictionary(kvp => kvp.Name, kvp => kvp); - config.PrivateRepositories = privateRepositories.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + config.PrivateRepositories = privateRepositories + .Where(r => !r.Value.Skip) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); return config; } catch (Exception e) @@ -69,6 +88,12 @@ private static TRepository RepositoryDefaults(TRepository r, string GitReferenceNext = string.IsNullOrEmpty(repository.GitReferenceNext) ? "main" : repository.GitReferenceNext, GitReferenceEdge = string.IsNullOrEmpty(repository.GitReferenceEdge) ? "main" : repository.GitReferenceEdge, }; + // ensure we always null path if we are running in CI + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI"))) + repository = repository with + { + Path = null + }; if (string.IsNullOrEmpty(repository.Origin)) { if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS"))) @@ -124,21 +149,34 @@ public ContentSourceMatch Match(string repository, string branchOrTag) var next = r.GetBranch(ContentSource.Next); var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag); if (current == branchOrTag) - match = match with { Current = ContentSource.Current }; + match = match with + { + Current = ContentSource.Current + }; if (next == branchOrTag) - match = match with { Next = ContentSource.Next }; + match = match with + { + Next = ContentSource.Next + }; if (isVersionBranch && SemVersion.TryParse(branchOrTag + ".0", out var v)) { // if the current branch is a version, only speculatively match if branch is actually a new version if (SemVersion.TryParse(current + ".0", out var currentVersion)) { if (v >= currentVersion) - match = match with { Speculative = true }; + match = match with + { + Speculative = true + }; } // assume we are newly onboarding the repository to current/next else - match = match with { Speculative = true }; + match = match with + { + Speculative = true + }; } + return match; } @@ -147,19 +185,27 @@ public ContentSourceMatch Match(string repository, string branchOrTag) // this is an unknown new elastic repository var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag); if (isVersionBranch || branchOrTag == "main" || branchOrTag == "master") - return match with { Speculative = true }; + return match with + { + Speculative = true + }; } if (Narrative.GetBranch(ContentSource.Current) == branchOrTag) - match = match with { Current = ContentSource.Current }; + match = match with + { + Current = ContentSource.Current + }; if (Narrative.GetBranch(ContentSource.Next) == branchOrTag) - match = match with { Next = ContentSource.Next }; + match = match with + { + Next = ContentSource.Next + }; return match; } public record ContentSourceMatch(ContentSource? Current, ContentSource? Next, bool Speculative); - } internal static partial class ContentSourceRegex diff --git a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs index 389fc257d..7b8d39440 100644 --- a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs +++ b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs @@ -36,6 +36,10 @@ public record Repository [YamlMember(Alias = "skip")] public bool Skip { get; set; } + /// Allows you to override the path to the repository, but only during local builds. + [YamlMember(Alias = "path")] + public string? Path { get; set; } + [YamlMember(Alias = "current")] public string GitReferenceCurrent { get; set; } = "main"; diff --git a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs index 045e6832f..a8cac2085 100644 --- a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs +++ b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs @@ -55,8 +55,9 @@ public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateReposit public IFileInfo LegacyUrlMappingsFile { get; } - public IFileInfo CreateNavigationFile(IReadOnlyDictionary privateRepositories) + public IFileInfo CreateNavigationFile(AssemblyConfiguration configuration) { + var privateRepositories = configuration.PrivateRepositories; if (privateRepositories.Count == 0 || !SkipPrivateRepositories) return NavigationFile; @@ -104,6 +105,23 @@ public IFileInfo CreateNavigationFile(IReadOnlyDictionary pr if (spacing == -1 || reindenting > 0) _fileSystem.File.AppendAllLines(tempFile, [line]); } + + if (configuration.AvailableRepositories.TryGetValue("docs-builder", out var docsBuildRepository) && docsBuildRepository is { Skip: false, Path: not null }) + { + // language=yaml + _fileSystem.File.AppendAllText(tempFile, + """ + - toc: docs-builder:// + path_prefix: reference/docs-builder + children: + - toc: docs-builder://development + path_prefix: reference/docs-builder/dev + children: + - toc: docs-builder://development/link-validation + path_prefix: reference/docs-builder/dev/link-val + + """); + } NavigationFile = _fileSystem.FileInfo.New(tempFile); return NavigationFile; diff --git a/src/Elastic.Documentation/GlobalCommandLine.cs b/src/Elastic.Documentation/GlobalCommandLine.cs index 9ebedb554..6de8a22a6 100644 --- a/src/Elastic.Documentation/GlobalCommandLine.cs +++ b/src/Elastic.Documentation/GlobalCommandLine.cs @@ -21,10 +21,9 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out i++; } else if (args[i] == "--skip-private-repositories") - { skipPrivateRepositories = true; - i++; - } + else if (args[i] == "--inject") + skipPrivateRepositories = true; else newArgs.Add(args[i]); } diff --git a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs index fe4dadfed..3d007f4d2 100644 --- a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs +++ b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs @@ -112,7 +112,8 @@ string CaptureOutput() WorkingDirectory = workingDirectory.FullName, Timeout = TimeSpan.FromSeconds(3), WaitForExit = TimeSpan.FromSeconds(3), - ConsoleOutWriter = NoopConsoleWriter.Instance + ConsoleOutWriter = NoopConsoleWriter.Instance, + OnlyPrintBinaryInExceptionMessage = false }; var result = Proc.Start(arguments); var line = (result.ExitCode, muteExceptions) switch diff --git a/src/tooling/docs-assembler/AssembleSources.cs b/src/tooling/docs-assembler/AssembleSources.cs index 01ba8846b..1bd8c1aa6 100644 --- a/src/tooling/docs-assembler/AssembleSources.cs +++ b/src/tooling/docs-assembler/AssembleSources.cs @@ -171,7 +171,7 @@ static void ReadHistoryMappings(IDictionary> public static FrozenDictionary GetTocMappings(AssembleContext context) { var dictionary = new Dictionary(); - var file = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); + var file = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration); var reader = new YamlStreamReader(file, context.Collector); var entries = new List>(); foreach (var entry in reader.Read()) diff --git a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs index 02781ec6e..01f53200c 100644 --- a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs @@ -78,12 +78,14 @@ public async Task CloneConfigurationFolder(string? gitRef = null, Cancel ct /// Treat warnings as errors and fail the build on warnings /// The environment to build /// If true, fetch the latest commit of the branch instead of the link registry entry ref + /// If true, assume the repository folder already exists on disk assume it's cloned already, primarily used for testing /// [Command("clone-all")] public async Task CloneAll( bool? strict = null, string? environment = null, bool? fetchLatest = null, + bool? assumeCloned = null, Cancel ctx = default ) { @@ -97,7 +99,7 @@ public async Task CloneAll( var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, fs, fs, null, null); var cloner = new AssemblerRepositorySourcer(logFactory, assembleContext); - _ = await cloner.CloneAll(fetchLatest ?? false, ctx); + _ = await cloner.CloneAll(fetchLatest ?? false, assumeCloned ?? false, ctx); await collector.StopAsync(ctx); diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs index 66c55bf0e..75f09a8ba 100644 --- a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -28,7 +28,7 @@ public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSou { _context = context; _assembleSources = assembleSources; - NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); + NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration); TableOfContents = Deserialize("toc"); Phantoms = Deserialize("phantoms"); ScopeDirectory = NavigationFile.Directory!; @@ -63,7 +63,7 @@ public static ImmutableHashSet GetPhantomPrefixes(AssembleContext context) private static ImmutableHashSet GetSourceUris(string key, AssembleContext context) { - var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); + var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration); var reader = new YamlStreamReader(navigationFile, context.Collector); var set = new HashSet(); foreach (var entry in reader.Read()) @@ -153,7 +153,7 @@ public void EmitError(string message) => private IReadOnlyCollection Deserialize(string key) { - var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration.PrivateRepositories); + var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration); var reader = new YamlStreamReader(navigationFile, _context.Collector); try { diff --git a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 94f4afb7b..4f943b430 100644 --- a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -35,6 +35,12 @@ public CheckoutResult GetAll() foreach (var repo in repositories.Values) { var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name)); + // if we are running locally, allow for repository path override + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) && !string.IsNullOrWhiteSpace(repo.Path)) + { + _logger.LogInformation("{RepositoryName}: Using local override path for {RepositoryName} at {Path}", repo.Name, repo.Name, repo.Path); + checkoutFolder = fs.DirectoryInfo.New(repo.Path); + } IGitRepository gitFacade = new SingleCommitOptimizedGitRepository(context.Collector, checkoutFolder); if (!checkoutFolder.Exists) { @@ -57,7 +63,7 @@ public CheckoutResult GetAll() }; } - public async Task CloneAll(bool fetchLatest, Cancel ctx = default) + public async Task CloneAll(bool fetchLatest, bool assumeCloned, Cancel ctx = default) { _logger.LogInformation("Cloning all repositories for environment {EnvironmentName} using '{ContentSourceStrategy}' content sourcing strategy", PublishEnvironment.Name, @@ -93,7 +99,9 @@ await Task.Run(() => } gitRef = entryInfo.GitReference; } - checkouts.Add(RepositorySourcer.CloneRef(repo.Value, gitRef, fetchLatest)); + + var cloneInformation = RepositorySourcer.CloneRef(repo.Value, gitRef, fetchLatest, assumeCloned: assumeCloned); + checkouts.Add(cloneInformation); }, c); }).ConfigureAwait(false); await context.WriteFileSystem.File.WriteAllTextAsync( @@ -125,22 +133,40 @@ public class RepositorySourcer(ILoggerFactory logFactory, IDirectoryInfo checkou // // The repository to clone. // The git reference to check out. Branch, commit or tag - public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true) + public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true, bool assumeCloned = false) { var checkoutFolder = - appendRepositoryName + appendRepositoryName ? readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name)) : checkoutDirectory; + + // if we are running locally, allow for repository path override + if (!string.IsNullOrWhiteSpace(repository.Path)) + { + var di = readFileSystem.DirectoryInfo.New(repository.Path); + if (!di.Exists) + { + _logger.LogInformation("{RepositoryName}: Can not find {RepositoryName}@{Commit} at local override path {CheckoutFolder}", repository.Name, repository.Name, gitRef, di.FullName); + collector.EmitError("", $"Can not find {repository.Name}@{gitRef} at local override path {di.FullName}"); + return new Checkout { Directory = di, HeadReference = "", Repository = repository }; + } + checkoutFolder = di; + assumeCloned = true; + _logger.LogInformation("{RepositoryName}: Using override path for {RepositoryName}@{Commit} at {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName); + } + IGitRepository git = new SingleCommitOptimizedGitRepository(collector, checkoutFolder); + + if (assumeCloned && checkoutFolder.Exists) + { + _logger.LogInformation("{RepositoryName}: Assuming {RepositoryName}@{Commit} is already checked out to {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName); + return new Checkout { Directory = checkoutFolder, HeadReference = git.GetCurrentCommit(), Repository = repository }; + } + if (attempt > 3) { collector.EmitError("", $"Failed to clone repository {repository.Name}@{gitRef} after 3 attempts"); - return new Checkout - { - Directory = checkoutFolder, - HeadReference = "", - Repository = repository, - }; + return new Checkout { Directory = checkoutFolder, HeadReference = "", Repository = repository }; } _logger.LogInformation("{RepositoryName}: Cloning repository {RepositoryName}@{Commit} to {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName); diff --git a/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs b/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs index a900b1975..ad5a02944 100644 --- a/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs +++ b/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs @@ -2,18 +2,40 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Configuration; using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Testing; using Elastic.Documentation.ServiceDefaults; using FluentAssertions; using InMemLogger; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using static Elastic.Documentation.Aspire.ResourceNames; +using ConfigurationManager = Microsoft.Extensions.Configuration.ConfigurationManager; [assembly: CaptureConsole, AssemblyFixture(typeof(Elastic.Assembler.IntegrationTests.DocumentationFixture))] namespace Elastic.Assembler.IntegrationTests; +public static partial class DistributedApplicationExtensions +{ + /// + /// Ensures all parameters in the application configuration have values set. + /// + public static TBuilder WithEmptyParameters(this TBuilder builder) + where TBuilder : IDistributedApplicationTestingBuilder + { + var parameters = builder.Resources.OfType().Where(p => !p.IsConnectionString).ToList(); + foreach (var parameter in parameters) + builder.Configuration[$"Parameters:{parameter.Name}"] = string.Empty; + + builder.Configuration[$"Parameters:DocumentationElasticUrl"] = "http://localhost.example:9200"; + return builder; + } +} + + public class DocumentationFixture : IAsyncLifetime { public DistributedApplication DistributedApplication { get; private set; } = null!; @@ -23,23 +45,64 @@ public class DocumentationFixture : IAsyncLifetime /// public async ValueTask InitializeAsync() { - var builder = await DistributedApplicationTestingBuilder.CreateAsync( - ["--skip-private-repositories"], + var builder = await DistributedApplicationTestingBuilder.CreateAsync( + ["--skip-private-repositories", "--assume-cloned"], (options, settings) => { options.DisableDashboard = true; options.AllowUnsecuredTransport = true; + options.EnableResourceLogging = true; } ); + _ = builder.WithEmptyParameters(); _ = builder.Services.AddElasticDocumentationLogging(LogLevel.Information); _ = builder.Services.AddLogging(c => c.AddXUnit()); _ = builder.Services.AddLogging(c => c.AddInMemory()); - // TODO expose this as secrets for now not needed integration tests - _ = builder.AddParameter("LlmGatewayUrl", ""); - _ = builder.AddParameter("LlmGatewayServiceAccountPath", ""); + + DistributedApplication = await builder.BuildAsync(); InMemoryLogger = DistributedApplication.Services.GetService()!; - await DistributedApplication.StartAsync(); + _ = DistributedApplication.StartAsync().WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken); + + _ = await DistributedApplication.ResourceNotifications + .WaitForResourceAsync(AssemblerClone, KnownResourceStates.TerminalStates, cancellationToken: TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken); + + await ValidateExitCode(AssemblerClone); + + _ = await DistributedApplication.ResourceNotifications + .WaitForResourceAsync(AssemblerBuild, KnownResourceStates.TerminalStates, cancellationToken: TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken); + + await ValidateExitCode(AssemblerBuild); + + try + { + _ = await DistributedApplication.ResourceNotifications + .WaitForResourceHealthyAsync(AssemblerServe, cancellationToken: TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromMinutes(1), TestContext.Current.CancellationToken); + } + catch (Exception e) + { + await DistributedApplication.StopAsync(); + await DistributedApplication.DisposeAsync(); + throw new Exception($"{e.Message}: {string.Join(Environment.NewLine, InMemoryLogger.RecordedLogs.Reverse().Take(30).Reverse())}", e); + } + } + + private async ValueTask ValidateExitCode(string resourceName) + { + var eventResource = await DistributedApplication.ResourceNotifications.WaitForResourceAsync(resourceName, _ => true); + var id = eventResource.ResourceId; + if (!DistributedApplication.ResourceNotifications.TryGetCurrentState(id, out var e)) + throw new Exception($"Could not find {resourceName} in the current state"); + if (e.Snapshot.ExitCode is not 0) + { + await DistributedApplication.StopAsync(); + await DistributedApplication.DisposeAsync(); + throw new Exception( + $"Exit code should be 0 for {resourceName}: {string.Join(Environment.NewLine, InMemoryLogger.RecordedLogs.Reverse().Take(30).Reverse())}"); + } } /// @@ -56,9 +119,7 @@ public class ServeStaticTests(DocumentationFixture fixture, ITestOutputHelper ou [Fact] public async Task AssertRequestToRootReturnsData() { - _ = await fixture.DistributedApplication.ResourceNotifications - .WaitForResourceHealthyAsync("DocsBuilderServeStatic", cancellationToken: TestContext.Current.CancellationToken); - var client = fixture.DistributedApplication.CreateHttpClient("DocsBuilderServeStatic", "http"); + var client = fixture.DistributedApplication.CreateHttpClient(AssemblerServe, "http"); var root = await client.GetStringAsync("/", TestContext.Current.CancellationToken); _ = root.Should().NotBeNullOrEmpty(); } diff --git a/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj b/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj index 563a928ac..fead12ee0 100644 --- a/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj +++ b/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj @@ -7,10 +7,10 @@ - - - - + + + + diff --git a/tests-integration/Elastic.Documentation.Aspire/AppHost.cs b/tests-integration/Elastic.Documentation.Aspire/AppHost.cs deleted file mode 100644 index a8e4e4d8f..000000000 --- a/tests-integration/Elastic.Documentation.Aspire/AppHost.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using Elastic.Documentation; -using Microsoft.Extensions.Logging; - -var logLevel = LogLevel.Information; -GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); -var globalArguments = new List(); -if (skipPrivateRepositories) - globalArguments.Add("--skip-private-repositories"); - -if (logLevel != LogLevel.Information) -{ - globalArguments.Add("--log-level"); - globalArguments.Add(logLevel.ToString()); -} - -var builder = DistributedApplication.CreateBuilder(args); - -// Add a secret parameter named "secret" -var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true); -var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true); - -var cloneAll = builder.AddProject("DocsAssemblerCloneAll").WithArgs(["repo", "clone-all", .. globalArguments]); - -var buildAll = builder.AddProject("DocsAssemblerBuildAll").WithArgs(["repo", "build-all", .. globalArguments]) - .WaitForCompletion(cloneAll); - -var elasticsearch = builder.AddElasticsearch("elasticsearch") - .WithEnvironment("LICENSE", "trial"); - -var api = builder.AddProject("ApiLambda").WithArgs(globalArguments) - .WithEnvironment("ENVIRONMENT", "dev") - .WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl) - .WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath) - .WaitFor(elasticsearch) - .WithReference(elasticsearch); - -var indexElasticsearch = builder.AddProject("DocsAssemblerElasticsearch") - .WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments]) - .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearch.GetEndpoint("http")) - .WithEnvironment(context => - { - context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearch.Resource.PasswordParameter; - }) - .WithReference(elasticsearch) - .WithExplicitStart() - .WaitFor(elasticsearch) - .WaitForCompletion(cloneAll); - -var indexElasticsearchSemantic = builder.AddProject("DocsAssemblerElasticsearchSemantic") - .WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments]) - .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearch.GetEndpoint("http")) - .WithEnvironment(context => - { - context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearch.Resource.PasswordParameter; - }) - .WithReference(elasticsearch) - .WithExplicitStart() - .WaitFor(elasticsearch) - .WaitForCompletion(cloneAll); - -var serveStatic = builder.AddProject("DocsBuilderServeStatic") - .WithReference(elasticsearch) - .WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl) - .WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath) - .WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearch.GetEndpoint("http")) - .WithEnvironment(context => - { - context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearch.Resource.PasswordParameter; - }) - .WithHttpEndpoint(port: 4000, isProxied: false) - .WithArgs(["serve-static", .. globalArguments]) - .WithHttpHealthCheck("/", 200) - .WaitFor(elasticsearch) - .WaitForCompletion(buildAll); - -builder.Build().Run(); diff --git a/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj b/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj deleted file mode 100644 index 44282aa75..000000000 --- a/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - Exe - net9.0 - enable - enable - 72f50f33-6fb9-4d08-bff3-39568fe370b3 - false - - - - - - - - - - - - - - -