Skip to content

Add Documentation Assembler GitHub Action #501

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 14, 2025
Merged

Add Documentation Assembler GitHub Action #501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions actions/assembler/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: 'Documentation Assembler'
description: 'Run the documenation assembler commands'

branding:
icon: 'filter'
color: 'red'
Comment on lines +4 to +6
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: This is not really required. And I think you can only ever see this if it was published in the marketplace


inputs:
command:
description: 'The assembler command to run'
required: true

runs:
using: 'docker'
image: "docker://ghcr.io/elastic/docs-assembler:edge"
6 changes: 6 additions & 0 deletions docs-builder.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "authoring", "tests\authorin
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Markdown.Refactor", "src\Elastic.Markdown.Refactor\Elastic.Markdown.Refactor.csproj", "{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler", "assembler", "{CFEE9FAD-9E0C-4C0E-A0C2-B97D594C14B5}"
ProjectSection(SolutionItems) = preProject
actions\assembler\action.yml = actions\assembler\action.yml
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -100,5 +105,6 @@ Global
{28350800-B44B-479B-86E2-1D39E321C0B4} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
{018F959E-824B-4664-B345-066784478D24} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
{CFEE9FAD-9E0C-4C0E-A0C2-B97D594C14B5} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
EndGlobalSection
EndGlobal
2 changes: 1 addition & 1 deletion src/docs-assembler/Cli/LinkCommands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ private void AssignOutputLogger()
/// Create an index.json file from all discovered links.json files in our S3 bucket
/// </summary>
/// <param name="ctx"></param>
[Command("links create-index")]
[Command("create-index")]
public async Task CreateLinkIndex(Cancel ctx = default)
{
AssignOutputLogger();
Expand Down
94 changes: 94 additions & 0 deletions src/docs-assembler/Cli/RepositoryCommands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

using System.Collections.Concurrent;
using System.Diagnostics;
using ConsoleAppFramework;
using Elastic.Markdown.IO;
using Microsoft.Extensions.Logging;
using ProcNet;
using ProcNet.Std;

namespace Documentation.Assembler.Cli;

public class ConsoleLineHandler(string prefix) : IConsoleLineHandler
{
public void Handle(LineOut lineOut) => lineOut.CharsOrString(
r => Console.Write(prefix + ": " + r),
l => Console.WriteLine(prefix + ": " + l));

public void Handle(Exception e) { }
}

internal class RepositoryCommands(ILoggerFactory logger)
{
private void AssignOutputLogger()
{
var log = logger.CreateLogger<Program>();
ConsoleApp.Log = msg => log.LogInformation(msg);
ConsoleApp.LogError = msg => log.LogError(msg);
}

// would love to use libgit2 so there is no git dependency but
// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
/// <summary> Clones all repositories </summary>
/// <param name="ctx"></param>
[Command("clone-all")]
public async Task CloneAll(Cancel ctx = default)
{
var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml");
var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile));

Console.WriteLine(config.Repositories.Count);
var dict = new ConcurrentDictionary<string, Stopwatch>();
await Parallel.ForEachAsync(config.Repositories,
new ParallelOptions { CancellationToken = ctx, MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, async (kv, c) =>
{
await Task.Run(() =>
{
var name = kv.Key;
var repository = kv.Value;
var checkoutFolder = Path.Combine(Paths.Root.FullName, $".artifacts/assembly/{name}");

var sw = Stopwatch.StartNew();
dict.AddOrUpdate(name, sw, (_, _) => sw);
Console.WriteLine($"Checkout: {name}\t{repository}\t{checkoutFolder}");
var branch = repository.Branch ?? "main";
var args = new StartArguments(
"git", "clone", repository.Origin, checkoutFolder, "--depth", "1"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about sparse-checkout? Did you already try that?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I followed the 4 clone patterns defined here: https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/

But can't remember the results :)

Will go over it again when we set up our runner.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup. In our case we can only download the docs folder.

, "--single-branch", "--branch", branch
);
Proc.StartRedirected(args, new ConsoleLineHandler(name));
sw.Stop();
}, c);
}).ConfigureAwait(false);

foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}");
}

/// <summary> List all checked out repositories </summary>
/// <param name="ctx"></param>
[Command("list")]
public async Task ListRepositories(Cancel ctx = default)
{
var assemblyPath = Path.Combine(Paths.Root.FullName, $".artifacts/assembly");
var dir = new DirectoryInfo(assemblyPath);
var dictionary = new Dictionary<string, string>();
foreach (var d in dir.GetDirectories())
{
var checkoutFolder = Path.Combine(assemblyPath, d.Name);

var capture = Proc.Start(
new StartArguments("git", "rev-parse", "--abbrev-ref", "HEAD") { WorkingDirectory = checkoutFolder }
);
dictionary.Add(d.Name, capture.ConsoleOut.FirstOrDefault()?.Line ?? "unknown");
}

foreach (var kv in dictionary.OrderBy(kv => kv.Value))
Console.WriteLine($"-> {kv.Key}\tbranch: {kv.Value}");

await Task.CompletedTask;
}
}
78 changes: 7 additions & 71 deletions src/docs-assembler/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Actions.Core.Extensions;
using Actions.Core.Services;
using ConsoleAppFramework;
using Documentation.Assembler;
using Documentation.Assembler.Cli;
Expand Down Expand Up @@ -38,81 +39,16 @@
await using var serviceProvider = services.BuildServiceProvider();
ConsoleApp.ServiceProvider = serviceProvider;


var app = ConsoleApp.Create();
app.UseFilter<StopwatchFilter>();
app.UseFilter<CatchExceptionFilter>();

app.Add<LinkCommands>();

// would love to use libgit2 so there is no git dependency but
// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
app.Add("clone-all", async Task (CancellationToken ctx) =>
{
Console.WriteLine(config.Repositories.Count);
var dict = new ConcurrentDictionary<string, Stopwatch>();
await Parallel.ForEachAsync(config.Repositories, new ParallelOptions
{
CancellationToken = ctx,
MaxDegreeOfParallelism = Environment.ProcessorCount / 4
}, async (kv, c) =>
{
await Task.Run(() =>
{
var name = kv.Key;
var repository = kv.Value;
var checkoutFolder = Path.Combine(Paths.Root.FullName, $".artifacts/assembly/{name}");
app.Add<LinkCommands>("link");
app.Add<RepositoryCommands>("repo");

var sw = Stopwatch.StartNew();
dict.AddOrUpdate(name, sw, (_, _) => sw);
Console.WriteLine($"Checkout: {name}\t{repository}\t{checkoutFolder}");
var branch = repository.Branch ?? "main";
var args = new StartArguments(
"git", "clone", repository.Origin, checkoutFolder, "--depth", "1"
, "--single-branch", "--branch", branch
);
Proc.StartRedirected(args, new ConsoleLineHandler(name));
sw.Stop();
}, c);
}).ConfigureAwait(false);

foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}");
});
app.Add("list", async Task (CancellationToken ctx) =>
{

var assemblyPath = Path.Combine(Paths.Root.FullName, $".artifacts/assembly");
var dir = new DirectoryInfo(assemblyPath);
var dictionary = new Dictionary<string, string>();
foreach (var d in dir.GetDirectories())
{
var checkoutFolder = Path.Combine(assemblyPath, d.Name);

var capture = Proc.Start(
new StartArguments("git", "rev-parse", "--abbrev-ref", "HEAD")
{
WorkingDirectory = checkoutFolder
}
);
dictionary.Add(d.Name, capture.ConsoleOut.FirstOrDefault()?.Line ?? "unknown");
}
foreach (var kv in dictionary.OrderBy(kv => kv.Value))
Console.WriteLine($"-> {kv.Key}\tbranch: {kv.Value}");

await Task.CompletedTask;
});
var githubActions = ConsoleApp.ServiceProvider.GetService<ICoreService>();
var command = githubActions?.GetInput("command");
if (!string.IsNullOrEmpty(command))
args = command.Split(' ');

await app.RunAsync(args);

namespace Documentation.Assembler
{
public class ConsoleLineHandler(string prefix) : IConsoleLineHandler
{
public void Handle(LineOut lineOut) => lineOut.CharsOrString(
r => Console.Write(prefix + ": " + r),
l => Console.WriteLine(prefix + ": " + l));

public void Handle(Exception e) { }
}
}