Skip to content

Make API generation non interactive #5078

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 9, 2020
Merged

Make API generation non interactive #5078

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7f75312
Improve API generation
Mpdreamz Nov 4, 2020
247bc59
clean up the output a tad more
Mpdreamz Nov 4, 2020
18ef909
there is no need for root.html to be stored to disk and checked in
Mpdreamz Nov 4, 2020
361e244
Use ref instead of branch
Mpdreamz Nov 6, 2020
7459151
update error message
Mpdreamz Nov 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/ApiGenerator/ApiGenerator.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<TargetFramework>netcoreapp3.0</TargetFramework>
<IsPackable>false</IsPackable>

<NoWarn>NU1701</NoWarn>
<NoWarn>CS1591;NU1701</NoWarn>
<PreserveCompilationContext>true</PreserveCompilationContext>
</PropertyGroup>
<ItemGroup>
Expand All @@ -15,6 +15,8 @@
<PackageReference Include="CsQuery.Core" Version="2.0.1" />
<!-- https://github.com/toddams/RazorLight/issues/172 -->
<PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.3" />
<PackageReference Include="Spectre.Console" Version="0.27.1-preview.0.8" />
<PackageReference Include="System.CommandLine.DragonFruit" Version="0.3.0-alpha.20371.2" />
<!--<PackageReference Include="RazorLight" Version="2.0.0-beta1" />-->
</ItemGroup>
<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion src/ApiGenerator/Configuration/GeneratorLocations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public static class GeneratorLocations
{
// @formatter:off — disable formatter after this line
public static string EsNetFolder { get; } = $@"{Root}../../src/Elasticsearch.Net/";
public static string LastDownloadedVersionFile { get; } = Path.Combine(Root, "last_downloaded_version.txt");
public static string LastDownloadedRef { get; } = Path.Combine(Root, "last_downloaded_version.txt");

public static string NestFolder { get; } = $@"{Root}../../src/Nest/";
public static string RestSpecificationFolder { get; } = $@"{Root}RestSpecification/";
Expand Down
8 changes: 7 additions & 1 deletion src/ApiGenerator/Generator/ApiEndpointFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,13 @@ private static void EnforceRequiredOnParts(string jsonFile, UrlInformation url)
{
var required = url.Paths.All(p => p.Path.Contains($"{{{part.Name}}}"));
if (part.Required != required)
ApiGenerator.Warnings.Add($"{jsonFile} has part: {part.Name} listed as {part.Required} but should be {required}");
{
var message = required
? "is [b green] required [/] but appears in spec as [b red] optional [/]"
: "is [b green] optional [/] but marked as [b red] required [/] ";
// TODO submit PR to fix these, too noisy for now
//ApiGenerator.Warnings.Add($"[grey]{jsonFile}[/] part [b white] {part.Name} [/] {message}");
}
part.Required = required;
}
}
Expand Down
30 changes: 9 additions & 21 deletions src/ApiGenerator/Generator/ApiGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
using ApiGenerator.Domain.Specification;
Expand All @@ -20,17 +21,17 @@ public class ApiGenerator
{
public static List<string> Warnings { get; private set; } = new List<string>();

public static async Task Generate(string downloadBranch, bool lowLevelOnly, RestApiSpec spec)
public static async Task Generate(string downloadBranch, bool lowLevelOnly, RestApiSpec spec, CancellationToken token)
{
static async Task DoGenerate(ICollection<RazorGeneratorBase> generators, RestApiSpec restApiSpec, bool highLevel)
static async Task DoGenerate(ICollection<RazorGeneratorBase> generators, RestApiSpec restApiSpec, bool highLevel, CancellationToken token)
{
var pbarOpts = new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray };
var pbarOpts = new ProgressBarOptions { ProgressCharacter = '─', BackgroundColor = ConsoleColor.Yellow };
var message = $"Generating {(highLevel ? "high" : "low")} level code";
using var pbar = new ProgressBar(generators.Count, message, pbarOpts);
foreach (var generator in generators)
{
pbar.Message = "Generating " + generator.Title;
await generator.Generate(restApiSpec, pbar);
await generator.Generate(restApiSpec, pbar, token);
pbar.Tick("Generated " + generator.Title);
}
}
Expand All @@ -55,23 +56,10 @@ static async Task DoGenerate(ICollection<RazorGeneratorBase> generators, RestApi
new RequestsGenerator(),
};

await DoGenerate(lowLevelGenerators, spec, highLevel: false);
await DoGenerate(lowLevelGenerators, spec, highLevel: false, token);
if (!lowLevelOnly)
await DoGenerate(highLevelGenerators, spec, highLevel: true);
await DoGenerate(highLevelGenerators, spec, highLevel: true, token);

// Check if there are any non-Stable endpoints present.
foreach (var endpoint in spec.Endpoints)
{
if (endpoint.Value.Stability != Stability.Stable)
Warnings.Add($"Endpoint {endpoint.Value.Name} is not marked as Stable ({endpoint.Value.Stability})");
}

if (Warnings.Count == 0) return;

Console.ForegroundColor = ConsoleColor.Yellow;
foreach (var warning in Warnings.Distinct().OrderBy(w => w))
Console.WriteLine(warning);
Console.ResetColor();
}

public static RestApiSpec CreateRestApiSpecModel(string downloadBranch, params string[] folders)
Expand All @@ -84,7 +72,7 @@ public static RestApiSpec CreateRestApiSpecModel(string downloadBranch, params s
var endpoints = new SortedDictionary<string, ApiEndpoint>();
var seenFiles = new HashSet<string>();
using (var pbar = new ProgressBar(directories.Count, $"Listing {directories.Count} directories",
new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray, CollapseWhenFinished = false }))
new ProgressBarOptions { ProgressCharacter = '─', BackgroundColor = ConsoleColor.DarkGray, CollapseWhenFinished = false }))
{
var folderFiles = directories.Select(dir =>
Directory.GetFiles(dir)
Expand Down
5 changes: 3 additions & 2 deletions src/ApiGenerator/Generator/Razor/ApiUrlsLookupsGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -13,12 +14,12 @@ public class ApiUrlsLookupsGenerator : RazorGeneratorBase
{
public override string Title => "NEST static url lookups";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
var view = ViewLocations.HighLevel("Requests", "ApiUrlsLookup.cshtml");
var target = GeneratorLocations.HighLevel("_Generated", "ApiUrlsLookup.generated.cs");

await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);
}
}
}
7 changes: 4 additions & 3 deletions src/ApiGenerator/Generator/Razor/DescriptorsGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -15,20 +16,20 @@ public class DescriptorsGenerator : RazorGeneratorBase
{
public override string Title => "NEST descriptors";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
// Delete existing files
foreach (var file in Directory.GetFiles(GeneratorLocations.NestFolder, "Descriptors.*.cs"))
File.Delete(file);

var view = ViewLocations.HighLevel("Descriptors", "RequestDescriptorBase.cshtml");
var target = GeneratorLocations.HighLevel("Descriptors.cs");
await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);

var dependantView = ViewLocations.HighLevel("Descriptors", "Descriptors.cshtml");
string Target(string id) => GeneratorLocations.HighLevel($"Descriptors.{id}.cs");
var namespaced = spec.EndpointsPerNamespaceHighLevel.ToList();
await DoRazorDependantFiles(progressBar, namespaced, dependantView, kv => kv.Key, id => Target(id));
await DoRazorDependantFiles(progressBar, namespaced, dependantView, kv => kv.Key, id => Target(id), token);
}
}
}
5 changes: 3 additions & 2 deletions src/ApiGenerator/Generator/Razor/EnumsGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -13,12 +14,12 @@ public class EnumsGenerator : RazorGeneratorBase
{
public override string Title => "Elasticsearch.Net enums";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
var view = ViewLocations.LowLevel("Enums.Generated.cshtml");
var target = GeneratorLocations.LowLevel("Api", "Enums.Generated.cs");

await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);
}
}
}
7 changes: 4 additions & 3 deletions src/ApiGenerator/Generator/Razor/HighLevelClientImplementationGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -16,21 +17,21 @@ public class HighLevelClientImplementationGenerator : RazorGeneratorBase
{
public override string Title => "NEST client implementation";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
// Delete existing files
foreach (var file in Directory.GetFiles(GeneratorLocations.NestFolder, "ElasticClient.*.cs"))
File.Delete(file);

var view = ViewLocations.HighLevel("Client", "Implementation", "ElasticClient.cshtml");
var target = GeneratorLocations.HighLevel($"ElasticClient.{CsharpNames.RootNamespace}.cs");
await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);

string Target(string id) => GeneratorLocations.HighLevel($"ElasticClient.{id}.cs");

var namespaced = spec.EndpointsPerNamespaceHighLevel.Where(kv => kv.Key != CsharpNames.RootNamespace).ToList();
var dependantView = ViewLocations.HighLevel("Client", "Implementation", "ElasticClient.Namespace.cshtml");
await DoRazorDependantFiles(progressBar, namespaced, dependantView, kv => kv.Key, id => Target(id));
await DoRazorDependantFiles(progressBar, namespaced, dependantView, kv => kv.Key, id => Target(id), token);

}
}
Expand Down
5 changes: 3 additions & 2 deletions src/ApiGenerator/Generator/Razor/HighLevelClientInterfaceGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -13,12 +14,12 @@ public class HighLevelClientInterfaceGenerator : RazorGeneratorBase
{
public override string Title => "NEST client interface";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
var view = ViewLocations.HighLevel("Client", "Interface", "IElasticClient.cshtml");
var target = GeneratorLocations.HighLevel("IElasticClient.Generated.cs");

await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);
}
}
}
7 changes: 4 additions & 3 deletions src/ApiGenerator/Generator/Razor/LowLevelClientImplementationGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -16,20 +17,20 @@ public class LowLevelClientImplementationGenerator : RazorGeneratorBase
{
public override string Title { get; } = "Elasticsearch.Net client implementation";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
// Delete existing files
foreach (var file in Directory.GetFiles(GeneratorLocations.EsNetFolder, "ElasticLowLevelClient.*.cs"))
File.Delete(file);

var view = ViewLocations.LowLevel("Client", "Implementation", "ElasticLowLevelClient.cshtml");
var target = GeneratorLocations.LowLevel($"ElasticLowLevelClient.{CsharpNames.RootNamespace}.cs");
await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);

var namespaced = spec.EndpointsPerNamespaceLowLevel.Where(kv => kv.Key != CsharpNames.RootNamespace).ToList();
var namespacedView = ViewLocations.LowLevel("Client", "Implementation", "ElasticLowLevelClient.Namespace.cshtml");
await DoRazorDependantFiles(progressBar, namespaced, namespacedView, kv => kv.Key,
id => GeneratorLocations.LowLevel($"ElasticLowLevelClient.{id}.cs"));
id => GeneratorLocations.LowLevel($"ElasticLowLevelClient.{id}.cs"), token);
}
}
}
5 changes: 3 additions & 2 deletions src/ApiGenerator/Generator/Razor/LowLevelClientInterfaceGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -13,12 +14,12 @@ public class LowLevelClientInterfaceGenerator : RazorGeneratorBase
{
public override string Title { get; } = "Elasticsearch.Net client interface";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
var view = ViewLocations.LowLevel("Client", "Interface", "IElasticLowLevelClient.cshtml");
var target = GeneratorLocations.LowLevel("IElasticLowLevelClient.Generated.cs");

await DoRazor(spec, view, target);
await DoRazor(spec, view, target, null, token);
}
}
}
20 changes: 14 additions & 6 deletions src/ApiGenerator/Generator/Razor/RazorGeneratorBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -24,12 +25,13 @@ public abstract class RazorGeneratorBase
.UseMemoryCachingProvider()
.Build();

protected async Task DoRazor<TModel>(TModel model, string viewLocation, string targetLocation, string cacheNameSuffix = null)
protected async Task DoRazor<TModel>(TModel model, string viewLocation, string targetLocation, string cacheNameSuffix, CancellationToken token)
{
try
{
var name = GetType().Name + cacheNameSuffix;
var sourceFileContents = File.ReadAllText(viewLocation);
var sourceFileContents = await File.ReadAllTextAsync(viewLocation, token);
token.ThrowIfCancellationRequested();
var generated = await Engine.CompileRenderStringAsync(name, sourceFileContents, model);
WriteFormattedCsharpFile(targetLocation, generated);
}
Expand All @@ -42,15 +44,21 @@ protected async Task DoRazor<TModel>(TModel model, string viewLocation, string t

protected async Task DoRazorDependantFiles<TModel>(
ProgressBar pbar, IReadOnlyCollection<TModel> items, string viewLocation,
Func<TModel, string> identifier, Func<string, string> target)
Func<TModel, string> identifier, Func<string, string> target,
CancellationToken token
)
{
using (var c = pbar.Spawn(items.Count, "Generating namespaces", new ProgressBarOptions { ForegroundColor = ConsoleColor.Yellow }))
using (var c = pbar.Spawn(items.Count, "Generating namespaces", new ProgressBarOptions
{
ProgressCharacter = '─',
ForegroundColor = ConsoleColor.Yellow
}))
{
foreach (var item in items)
{
var id = identifier(item);
var targetLocation = target(id);
await DoRazor(item, viewLocation, targetLocation, id);
await DoRazor(item, viewLocation, targetLocation, id, token);
c.Tick($"{Title}: {id}");
}
}
Expand All @@ -65,6 +73,6 @@ protected static void WriteFormattedCsharpFile(string path, string contents)
}

public abstract string Title { get; }
public abstract Task Generate(RestApiSpec spec, ProgressBar progressBar);
public abstract Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token);
}
}
5 changes: 3 additions & 2 deletions src/ApiGenerator/Generator/Razor/RequestParametersGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ApiGenerator.Configuration;
using ApiGenerator.Domain;
Expand All @@ -15,7 +16,7 @@ public class RequestParametersGenerator : RazorGeneratorBase
{
public override string Title { get; } = "Elasticsearch.Net request parameters";

public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
{
// Delete existing files
foreach (var file in Directory.GetFiles(GeneratorLocations.EsNetFolder, "RequestParameters.*.cs"))
Expand All @@ -25,7 +26,7 @@ public override async Task Generate(RestApiSpec spec, ProgressBar progressBar)
string Target(string id) => GeneratorLocations.LowLevel("Api", "RequestParameters", $"RequestParameters.{id}.cs");

var namespaced = spec.EndpointsPerNamespaceLowLevel.ToList();
await DoRazorDependantFiles(progressBar, namespaced, view, kv => kv.Key, id => Target(id));
await DoRazorDependantFiles(progressBar, namespaced, view, kv => kv.Key, id => Target(id), token);
}
}
}
Loading