diff --git a/src/tooling/docs-assembler/Cli/DeployCommands.cs b/src/tooling/docs-assembler/Cli/DeployCommands.cs index cf40353ca..94e3e4a72 100644 --- a/src/tooling/docs-assembler/Cli/DeployCommands.cs +++ b/src/tooling/docs-assembler/Cli/DeployCommands.cs @@ -40,6 +40,7 @@ private void AssignOutputLogger() /// The file to write the plan to /// The percentage of deletions allowed in the plan as percentage of total files to sync /// + [Command("plan")] public async Task Plan( string environment, string s3BucketName, @@ -58,7 +59,7 @@ public async Task Plan( var s3Client = new AmazonS3Client(); var planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext); var plan = await planner.Plan(deleteThreshold, ctx); - _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests); + _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted); _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); @@ -70,7 +71,7 @@ public async Task Plan( if (!validationResult.Valid) { await githubActionsService.SetOutputAsync("plan-valid", "false"); - collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions"); + collector.EmitError(@out, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}"); await collector.StopAsync(ctx); return collector.Errors; } @@ -93,6 +94,7 @@ public async Task Plan( /// The S3 bucket name to deploy to /// The path to the plan file to apply /// + [Command("apply")] public async Task Apply(string environment, string s3BucketName, string planFile, Cancel ctx = default) { AssignOutputLogger(); @@ -116,6 +118,7 @@ public async Task Apply(string environment, string s3BucketName, string pla } var planJson = await File.ReadAllTextAsync(planFile, ctx); var plan = SyncPlan.Deserialize(planJson); + _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted); _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests); _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); @@ -133,7 +136,7 @@ public async Task Apply(string environment, string s3BucketName, string pla var validationResult = validator.Validate(plan); if (!validationResult.Valid) { - collector.EmitError(planFile, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions"); + collector.EmitError(planFile, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}"); await collector.StopAsync(ctx); return collector.Errors; } diff --git a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs index 509d3848c..2af584f4e 100644 --- a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs @@ -92,7 +92,7 @@ private bool IsSymlink(string path) public async Task Plan(float? deleteThreshold, Cancel ctx = default) { - var remoteObjects = await ListObjects(ctx); + var (readToCompletion, remoteObjects) = await ListObjects(ctx); var localObjects = context.OutputDirectory.GetFiles("*", SearchOption.AllDirectories) .Where(f => !IsSymlink(f.FullName)) .ToArray(); @@ -156,6 +156,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) => return new SyncPlan { + RemoteListingCompleted = readToCompletion, DeleteThresholdDefault = deleteThreshold, TotalRemoteFiles = remoteObjects.Count, TotalSourceFiles = localObjects.Length, @@ -167,24 +168,56 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) => }; } - private async Task> ListObjects(Cancel ctx = default) + private async Task<(bool readToCompletion, Dictionary objects)> ListObjects(Cancel ctx = default) { var listBucketRequest = new ListObjectsV2Request { BucketName = bucketName, - MaxKeys = 1000, + MaxKeys = 1000 }; var objects = new List(); + var bucketExists = await S3BucketExists(ctx); + if (!bucketExists) + { + context.Collector.EmitGlobalError("Bucket does not exist, cannot list objects"); + return (false, objects.ToDictionary(o => o.Key)); + } + + var readToCompletion = true; ListObjectsV2Response response; do { response = await s3Client.ListObjectsV2Async(listBucketRequest, ctx); if (response is null or { S3Objects: null }) + { + if (response?.IsTruncated == true) + { + context.Collector.EmitGlobalError("Failed to list objects in S3 to completion"); + readToCompletion = false; + } break; + } objects.AddRange(response.S3Objects); listBucketRequest.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated == true); - return objects.ToDictionary(o => o.Key); + return (readToCompletion, objects.ToDictionary(o => o.Key)); + } + + private async Task S3BucketExists(Cancel ctx) + { + //https://docs.aws.amazon.com/code-library/latest/ug/s3_example_s3_Scenario_DoesBucketExist_section.html + try + { + _ = await s3Client.GetBucketAclAsync(new GetBucketAclRequest + { + BucketName = bucketName + }, ctx); + return true; + } + catch + { + return false; + } } } diff --git a/src/tooling/docs-assembler/Deploying/DocsSync.cs b/src/tooling/docs-assembler/Deploying/DocsSync.cs index 0a537fb44..ed9504b4e 100644 --- a/src/tooling/docs-assembler/Deploying/DocsSync.cs +++ b/src/tooling/docs-assembler/Deploying/DocsSync.cs @@ -55,6 +55,10 @@ public record SyncPlan [JsonPropertyName("deletion_threshold_default")] public required float? DeleteThresholdDefault { get; init; } + /// The user-specified delete threshold + [JsonPropertyName("remote_listing_completed")] + public required bool RemoteListingCompleted { get; init; } + /// The total number of source files that were located in the build output [JsonPropertyName("total_source_files")] public required int TotalSourceFiles { get; init; } diff --git a/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs b/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs index 1477bb3c4..cb5500f8e 100644 --- a/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs +++ b/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs @@ -16,6 +16,12 @@ public PlanValidationResult Validate(SyncPlan plan) _logger.LogInformation("Using user-specified delete threshold of {Threshold}", plan.DeleteThresholdDefault); var deleteThreshold = plan.DeleteThresholdDefault ?? 0.2f; + if (!plan.RemoteListingCompleted) + { + _logger.LogError("Remote files were not read to completion, cannot validate deployment plan"); + return new(false, 1.0f, deleteThreshold); + } + if (plan.TotalSourceFiles == 0) { _logger.LogError("No files to sync"); @@ -30,7 +36,7 @@ public PlanValidationResult Validate(SyncPlan plan) } // if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8 // this allows newer assembled documentation to be in a higher state of flux - if (plan.TotalRemoteFiles <= 100) + else if (plan.TotalRemoteFiles <= 100) { _logger.LogInformation("Plan has less than 100 total remote files ensuring delete threshold is at minimum 0.8"); deleteThreshold = Math.Max(deleteThreshold, 0.8f); diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs index 9ea21c70e..1d8bc388f 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs @@ -235,6 +235,7 @@ public async Task TestApply() var context = new AssembleContext(config, configurationContext, "dev", collector, fileSystem, fileSystem, null, checkoutDirectory); var plan = new SyncPlan { + RemoteListingCompleted = true, DeleteThresholdDefault = null, TotalRemoteFiles = 0, TotalSourceFiles = 5,