diff --git a/src/tooling/docs-assembler/Cli/DeployCommands.cs b/src/tooling/docs-assembler/Cli/DeployCommands.cs
index cf40353ca..94e3e4a72 100644
--- a/src/tooling/docs-assembler/Cli/DeployCommands.cs
+++ b/src/tooling/docs-assembler/Cli/DeployCommands.cs
@@ -40,6 +40,7 @@ private void AssignOutputLogger()
/// The file to write the plan to
/// The percentage of deletions allowed in the plan as percentage of total files to sync
///
+ [Command("plan")]
public async Task Plan(
string environment,
string s3BucketName,
@@ -58,7 +59,7 @@ public async Task Plan(
var s3Client = new AmazonS3Client();
var planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
var plan = await planner.Plan(deleteThreshold, ctx);
- _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
+ _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted);
_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
_logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count);
@@ -70,7 +71,7 @@ public async Task Plan(
if (!validationResult.Valid)
{
await githubActionsService.SetOutputAsync("plan-valid", "false");
- collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
+ collector.EmitError(@out, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}");
await collector.StopAsync(ctx);
return collector.Errors;
}
@@ -93,6 +94,7 @@ public async Task Plan(
/// The S3 bucket name to deploy to
/// The path to the plan file to apply
///
+ [Command("apply")]
public async Task Apply(string environment, string s3BucketName, string planFile, Cancel ctx = default)
{
AssignOutputLogger();
@@ -116,6 +118,7 @@ public async Task Apply(string environment, string s3BucketName, string pla
}
var planJson = await File.ReadAllTextAsync(planFile, ctx);
var plan = SyncPlan.Deserialize(planJson);
+ _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted);
_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
@@ -133,7 +136,7 @@ public async Task Apply(string environment, string s3BucketName, string pla
var validationResult = validator.Validate(plan);
if (!validationResult.Valid)
{
- collector.EmitError(planFile, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
+ collector.EmitError(planFile, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}");
await collector.StopAsync(ctx);
return collector.Errors;
}
diff --git a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs
index 509d3848c..2af584f4e 100644
--- a/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs
+++ b/src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs
@@ -92,7 +92,7 @@ private bool IsSymlink(string path)
public async Task Plan(float? deleteThreshold, Cancel ctx = default)
{
- var remoteObjects = await ListObjects(ctx);
+ var (readToCompletion, remoteObjects) = await ListObjects(ctx);
var localObjects = context.OutputDirectory.GetFiles("*", SearchOption.AllDirectories)
.Where(f => !IsSymlink(f.FullName))
.ToArray();
@@ -156,6 +156,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
return new SyncPlan
{
+ RemoteListingCompleted = readToCompletion,
DeleteThresholdDefault = deleteThreshold,
TotalRemoteFiles = remoteObjects.Count,
TotalSourceFiles = localObjects.Length,
@@ -167,24 +168,56 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
};
}
- private async Task> ListObjects(Cancel ctx = default)
+ private async Task<(bool readToCompletion, Dictionary objects)> ListObjects(Cancel ctx = default)
{
var listBucketRequest = new ListObjectsV2Request
{
BucketName = bucketName,
- MaxKeys = 1000,
+ MaxKeys = 1000
};
var objects = new List();
+ var bucketExists = await S3BucketExists(ctx);
+ if (!bucketExists)
+ {
+ context.Collector.EmitGlobalError("Bucket does not exist, cannot list objects");
+ return (false, objects.ToDictionary(o => o.Key));
+ }
+
+ var readToCompletion = true;
ListObjectsV2Response response;
do
{
response = await s3Client.ListObjectsV2Async(listBucketRequest, ctx);
if (response is null or { S3Objects: null })
+ {
+ if (response?.IsTruncated == true)
+ {
+ context.Collector.EmitGlobalError("Failed to list objects in S3 to completion");
+ readToCompletion = false;
+ }
break;
+ }
objects.AddRange(response.S3Objects);
listBucketRequest.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated == true);
- return objects.ToDictionary(o => o.Key);
+ return (readToCompletion, objects.ToDictionary(o => o.Key));
+ }
+
+ private async Task S3BucketExists(Cancel ctx)
+ {
+ //https://docs.aws.amazon.com/code-library/latest/ug/s3_example_s3_Scenario_DoesBucketExist_section.html
+ try
+ {
+ _ = await s3Client.GetBucketAclAsync(new GetBucketAclRequest
+ {
+ BucketName = bucketName
+ }, ctx);
+ return true;
+ }
+ catch
+ {
+ return false;
+ }
}
}
diff --git a/src/tooling/docs-assembler/Deploying/DocsSync.cs b/src/tooling/docs-assembler/Deploying/DocsSync.cs
index 0a537fb44..ed9504b4e 100644
--- a/src/tooling/docs-assembler/Deploying/DocsSync.cs
+++ b/src/tooling/docs-assembler/Deploying/DocsSync.cs
@@ -55,6 +55,10 @@ public record SyncPlan
[JsonPropertyName("deletion_threshold_default")]
public required float? DeleteThresholdDefault { get; init; }
+ /// The user-specified delete threshold
+ [JsonPropertyName("remote_listing_completed")]
+ public required bool RemoteListingCompleted { get; init; }
+
/// The total number of source files that were located in the build output
[JsonPropertyName("total_source_files")]
public required int TotalSourceFiles { get; init; }
diff --git a/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs b/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs
index 1477bb3c4..cb5500f8e 100644
--- a/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs
+++ b/src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs
@@ -16,6 +16,12 @@ public PlanValidationResult Validate(SyncPlan plan)
_logger.LogInformation("Using user-specified delete threshold of {Threshold}", plan.DeleteThresholdDefault);
var deleteThreshold = plan.DeleteThresholdDefault ?? 0.2f;
+ if (!plan.RemoteListingCompleted)
+ {
+ _logger.LogError("Remote files were not read to completion, cannot validate deployment plan");
+ return new(false, 1.0f, deleteThreshold);
+ }
+
if (plan.TotalSourceFiles == 0)
{
_logger.LogError("No files to sync");
@@ -30,7 +36,7 @@ public PlanValidationResult Validate(SyncPlan plan)
}
// if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8
// this allows newer assembled documentation to be in a higher state of flux
- if (plan.TotalRemoteFiles <= 100)
+ else if (plan.TotalRemoteFiles <= 100)
{
_logger.LogInformation("Plan has less than 100 total remote files ensuring delete threshold is at minimum 0.8");
deleteThreshold = Math.Max(deleteThreshold, 0.8f);
diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs
index 9ea21c70e..1d8bc388f 100644
--- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs
+++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs
@@ -235,6 +235,7 @@ public async Task TestApply()
var context = new AssembleContext(config, configurationContext, "dev", collector, fileSystem, fileSystem, null, checkoutDirectory);
var plan = new SyncPlan
{
+ RemoteListingCompleted = true,
DeleteThresholdDefault = null,
TotalRemoteFiles = 0,
TotalSourceFiles = 5,