diff --git a/src/tooling/docs-assembler/Cli/DeployCommands.cs b/src/tooling/docs-assembler/Cli/DeployCommands.cs index 0d0fa96fd..1b04e7e1e 100644 --- a/src/tooling/docs-assembler/Cli/DeployCommands.cs +++ b/src/tooling/docs-assembler/Cli/DeployCommands.cs @@ -25,12 +25,13 @@ internal sealed class DeployCommands( ICoreService githubActionsService ) { + private readonly ILogger _logger = logFactory.CreateLogger(); + [SuppressMessage("Usage", "CA2254:Template should be a static expression")] private void AssignOutputLogger() { - var log = logFactory.CreateLogger(); - ConsoleApp.Log = msg => log.LogInformation(msg); - ConsoleApp.LogError = msg => log.LogError(msg); + ConsoleApp.Log = msg => _logger.LogInformation(msg); + ConsoleApp.LogError = msg => _logger.LogError(msg); } /// Creates a sync plan @@ -57,20 +58,17 @@ public async Task Plan( var s3Client = new AmazonS3Client(); IDocsSyncPlanStrategy planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext); var plan = await planner.Plan(ctx); - ConsoleApp.Log("Total files to sync: " + plan.TotalSyncRequests); - ConsoleApp.Log("Total files to delete: " + plan.DeleteRequests.Count); - ConsoleApp.Log("Total files to add: " + plan.AddRequests.Count); - ConsoleApp.Log("Total files to update: " + plan.UpdateRequests.Count); - ConsoleApp.Log("Total files to skip: " + plan.SkipRequests.Count); - if (plan.TotalSyncRequests == 0) - { - collector.EmitError(@out, $"Plan has no files to sync so no plan will be written."); - await collector.StopAsync(ctx); - return collector.Errors; - } + _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests); + _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); + _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); + _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); + _logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count); + _logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles); + _logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles); var validationResult = planner.Validate(plan, deleteThreshold); if (!validationResult.Valid) { + await githubActionsService.SetOutputAsync("plan-valid", "false"); collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalSyncRequests:N0} files while plan has {plan.DeleteRequests:N0} deletions"); await collector.StopAsync(ctx); return collector.Errors; @@ -85,6 +83,7 @@ public async Task Plan( ConsoleApp.Log("Plan written to " + @out); } await collector.StopAsync(ctx); + await githubActionsService.SetOutputAsync("plan-valid", collector.Errors == 0 ? "true" : "false"); return collector.Errors; } @@ -121,6 +120,19 @@ public async Task Apply( } var planJson = await File.ReadAllTextAsync(planFile, ctx); var plan = SyncPlan.Deserialize(planJson); + _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests); + _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); + _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); + _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); + _logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count); + _logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles); + _logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles); + if (plan.TotalSyncRequests == 0) + { + _logger.LogInformation("Plan has no files to sync, skipping incremental synchronization"); + await collector.StopAsync(ctx); + return collector.Errors; + } await applier.Apply(plan, ctx); await collector.StopAsync(ctx); return collector.Errors; diff --git a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs index cd5a260e2..d61320fec 100644 --- a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs @@ -158,6 +158,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) => return new SyncPlan { + TotalRemoteFiles = remoteObjects.Count, TotalSourceFiles = localObjects.Length, DeleteRequests = deleteRequests.ToList(), AddRequests = addRequests.ToList(), @@ -176,15 +177,20 @@ public PlanValidationResult Validate(SyncPlan plan, float deleteThreshold) return new(false, 1.0f, deleteThreshold); } - var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalSyncRequests; - // if the total sync requests are less than 100, we enforce a higher ratio of 0.8 + var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalRemoteFiles; + if (plan.TotalRemoteFiles == 0) + { + _logger.LogInformation("No files discovered in S3, assuming a clean bucket resetting delete threshold to `0.0' as our plan should not have ANY deletions"); + deleteThreshold = 0.0f; + } + // if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8 // this allows newer assembled documentation to be in a higher state of flux - if (plan.TotalSyncRequests <= 100) + if (plan.TotalRemoteFiles <= 100) deleteThreshold = Math.Max(deleteThreshold, 0.8f); - // if the total sync requests are less than 1000, we enforce a higher ratio of 0.5 + // if the total remote files are less than or equal to 1000, we enforce a higher ratio of 0.5 // this allows newer assembled documentation to be in a higher state of flux - else if (plan.TotalSyncRequests <= 1000) + else if (plan.TotalRemoteFiles <= 1000) deleteThreshold = Math.Max(deleteThreshold, 0.5f); if (deleteRatio > deleteThreshold) diff --git a/src/tooling/docs-assembler/Deploying/DocsSync.cs b/src/tooling/docs-assembler/Deploying/DocsSync.cs index 2ee8634bd..cdc0b7d69 100644 --- a/src/tooling/docs-assembler/Deploying/DocsSync.cs +++ b/src/tooling/docs-assembler/Deploying/DocsSync.cs @@ -52,9 +52,15 @@ public record SkipRequest : SyncRequest public record SyncPlan { + /// The total number of source files that were located in the build output [JsonPropertyName("total_source_files")] public required int TotalSourceFiles { get; init; } + /// The total number of remote files that were located in the remote location + [JsonPropertyName("total_remote_files")] + public required int TotalRemoteFiles { get; init; } + + /// The total number of sync requests that were generated (sum of , , ) [JsonPropertyName("total_sync_requests")] public required int TotalSyncRequests { get; init; } diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs index 01f4c72fe..6d8a97992 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs @@ -66,6 +66,8 @@ public async Task TestPlan() // Assert + plan.TotalRemoteFiles.Should().Be(3); + plan.TotalSourceFiles.Should().Be(5); plan.TotalSyncRequests.Should().Be(6); //including skip on server @@ -231,6 +233,7 @@ public async Task TestApply() var context = new AssembleContext(config, configurationContext, "dev", collector, fileSystem, fileSystem, null, checkoutDirectory); var plan = new SyncPlan { + TotalRemoteFiles = 0, TotalSourceFiles = 5, TotalSyncRequests = 6, AddRequests = [