From c5304017feb13959b657c89a847499b06b1f7cdc Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Tue, 1 Jul 2025 19:59:41 -0300 Subject: [PATCH 1/9] Refactor communication with AWS for KeyValueStore updates --- Directory.Packages.props | 2 - .../ExternalCommandExecutor.cs | 9 +- .../docs-assembler/Cli/DeployCommands.cs | 76 +---------- .../AwsCloudFrontKeyValueStoreProxy.cs | 127 ++++++++++++++++++ .../AwsCloudFrontKeyValueStoreModels.cs | 37 +++++ .../docs-assembler/docs-assembler.csproj | 2 - 6 files changed, 171 insertions(+), 82 deletions(-) create mode 100644 src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs create mode 100644 src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 1f67f49a5..eea93b757 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,8 +13,6 @@ - - diff --git a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs index 363b60987..5c51e9940 100644 --- a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs +++ b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs @@ -12,6 +12,7 @@ namespace Elastic.Documentation.Tooling.ExternalCommands; public abstract class ExternalCommandExecutor(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) { protected IDirectoryInfo WorkingDirectory => workingDirectory; + protected DiagnosticsCollector Collector => collector; protected void ExecIn(Dictionary environmentVars, string binary, params string[] args) { var arguments = new ExecArguments(binary, args) @@ -77,13 +78,13 @@ string[] CaptureOutput() } - protected string Capture(string binary, params string[] args) => Capture(false, binary, args); - - protected string Capture(bool muteExceptions, string binary, params string[] args) + protected string Capture(string binary, params string[] args) => Capture(false, 10, binary, args); + protected string Capture(bool muteExceptions, string binary, params string[] args) => Capture(muteExceptions, 10, binary, args); + protected string Capture(bool muteExceptions, int attempts, string binary, params string[] args) { // Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try Exception? e = null; - for (var i = 0; i <= 9; i++) + for (var i = 1; i <= attempts; i++) { try { diff --git a/src/tooling/docs-assembler/Cli/DeployCommands.cs b/src/tooling/docs-assembler/Cli/DeployCommands.cs index 9ece2c445..fcf96d817 100644 --- a/src/tooling/docs-assembler/Cli/DeployCommands.cs +++ b/src/tooling/docs-assembler/Cli/DeployCommands.cs @@ -6,9 +6,6 @@ using System.IO.Abstractions; using System.Text.Json; using Actions.Core.Services; -using Amazon.CloudFront; -using Amazon.CloudFrontKeyValueStore; -using Amazon.CloudFrontKeyValueStore.Model; using Amazon.S3; using Amazon.S3.Transfer; using ConsoleAppFramework; @@ -20,12 +17,6 @@ namespace Documentation.Assembler.Cli; -internal enum KvsOperation -{ - Puts, - Deletes -} - internal sealed class DeployCommands(ILoggerFactory logger, ICoreService githubActionsService) { [SuppressMessage("Usage", "CA2254:Template should be a static expression")] @@ -144,74 +135,11 @@ public async Task UpdateRedirects( } var kvsName = $"elastic-docs-v3-{environment}-redirects-kvs"; + var cloudFrontClient = new AwsCloudFrontKeyValueStoreProxy(collector, new FileSystem().DirectoryInfo.New(Directory.GetCurrentDirectory())); - var cfClient = new AmazonCloudFrontClient(); - var kvsClient = new AmazonCloudFrontKeyValueStoreClient(); - - ConsoleApp.Log("Describing KVS"); - var describeResponse = await cfClient.DescribeKeyValueStoreAsync(new Amazon.CloudFront.Model.DescribeKeyValueStoreRequest { Name = kvsName }, ctx); - - var kvsArn = describeResponse.KeyValueStore.ARN; - var eTag = describeResponse.ETag; - var existingRedirects = new HashSet(); - - var listKeysRequest = new ListKeysRequest { KvsARN = kvsArn }; - ListKeysResponse listKeysResponse; - - do - { - listKeysResponse = await kvsClient.ListKeysAsync(listKeysRequest, ctx); - foreach (var item in listKeysResponse.Items) - _ = existingRedirects.Add(item.Key); - listKeysRequest.NextToken = listKeysResponse.NextToken; - } - while (!string.IsNullOrEmpty(listKeysResponse.NextToken)); - - var toPut = sourcedRedirects - .Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value }); - var toDelete = existingRedirects - .Except(sourcedRedirects.Keys) - .Select(k => new DeleteKeyRequestListItem { Key = k }); - - ConsoleApp.Log("Updating redirects in KVS"); - const int batchSize = 50; - - eTag = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toPut, batchSize, KvsOperation.Puts, ctx); - _ = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toDelete, batchSize, KvsOperation.Deletes, ctx); + cloudFrontClient.UpdateRedirects(kvsName, sourcedRedirects); await collector.StopAsync(ctx); return collector.Errors; } - - private static async Task ProcessBatchUpdatesAsync( - IAmazonCloudFrontKeyValueStore kvsClient, - string kvsArn, - string eTag, - IEnumerable items, - int batchSize, - KvsOperation operation, - Cancel ctx) - { - var enumerable = items.ToList(); - for (var i = 0; i < enumerable.Count; i += batchSize) - { - var batch = enumerable.Skip(i).Take(batchSize); - var updateRequest = new UpdateKeysRequest - { - KvsARN = kvsArn, - IfMatch = eTag - }; - - if (operation is KvsOperation.Puts) - updateRequest.Puts = batch.Cast().ToList(); - else if (operation is KvsOperation.Deletes) - updateRequest.Deletes = batch.Cast().ToList(); - - var update = await kvsClient.UpdateKeysAsync(updateRequest, ctx); - eTag = update.ETag; - } - - return eTag; - } - } diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs new file mode 100644 index 000000000..0d1f1a394 --- /dev/null +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text.Json; +using ConsoleAppFramework; +using Documentation.Assembler.Deploying.Serialization; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Tooling.ExternalCommands; + +namespace Documentation.Assembler.Deploying; + +internal enum KvsOperation +{ + Puts, + Deletes +} + +public class AwsCloudFrontKeyValueStoreProxy(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) : ExternalCommandExecutor(collector, workingDirectory) +{ + public void UpdateRedirects(string kvsName, IReadOnlyDictionary sourcedRedirects) + { + var (kvsArn, eTag) = DescribeKeyValueStore(kvsName); + if (string.IsNullOrEmpty(kvsArn) || string.IsNullOrEmpty(eTag)) + return; + + var existingRedirects = ListAllKeys(kvsArn); + + var toPut = sourcedRedirects + .Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value }); + var toDelete = existingRedirects + .Except(sourcedRedirects.Keys) + .Select(k => new DeleteKeyRequestListItem { Key = k }); + + eTag = ProcessBatchUpdates(kvsArn, eTag, toPut, KvsOperation.Puts); + _ = ProcessBatchUpdates(kvsArn, eTag, toDelete, KvsOperation.Deletes); + } + + private (string? Arn, string? ETag) DescribeKeyValueStore(string kvsName) + { + ConsoleApp.Log("Describing KeyValueStore"); + try + { + var json = Capture("aws", "cloudfront", "describe-key-value-store", "--name", kvsName); + var describeResponse = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); + if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 }) + return (describeResponse.KeyValueStore.ARN, describeResponse.ETag); + + Collector.EmitError("", "Could not deserialize the DescribeKeyValueStoreResponse"); + return (null, null); + } + catch (Exception e) + { + Collector.EmitError("", "An error occurred while describing the KeyValueStore", e); + return (null, null); + } + } + + private HashSet ListAllKeys(string kvsArn) + { + ConsoleApp.Log("Acquiring existing redirects"); + var allKeys = new HashSet(); + string[] baseArgs = ["cloudfront-key-value-store", "list-keys", "--kvs-arn", kvsArn]; + string? nextToken = null; + try + { + do + { + var json = Capture("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--next-token", nextToken] : []]); + var response = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.ListKeysResponse); + + if (response?.Items != null) + { + foreach (var item in response.Items) + _ = allKeys.Add(item.Key); + } + + nextToken = response?.NextToken; + } while (!string.IsNullOrEmpty(nextToken)); + } + catch (Exception e) + { + Collector.EmitError("", "An error occurred while acquiring existing redirects in the KeyValueStore", e); + return []; + } + return allKeys; + } + + + private string ProcessBatchUpdates( + string kvsArn, + string eTag, + IEnumerable items, + KvsOperation operation) + { + const int batchSize = 50; + ConsoleApp.Log($"Processing {items.Count()} items in batches of {batchSize} for {operation} update operation."); + try + { + foreach (var batch in items.Chunk(batchSize)) + { + var payload = operation switch + { + KvsOperation.Puts => JsonSerializer.Serialize(batch.Cast().ToList(), + AwsCloudFrontKeyValueStoreJsonContext.Default.ListPutKeyRequestListItem), + KvsOperation.Deletes => JsonSerializer.Serialize(batch.Cast().ToList(), + AwsCloudFrontKeyValueStoreJsonContext.Default.ListDeleteKeyRequestListItem), + _ => string.Empty + }; + var responseJson = Capture("aws", "cloudfront-key-value-store", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, + $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload); + var updateResponse = JsonSerializer.Deserialize(responseJson, AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); + + if (string.IsNullOrEmpty(updateResponse?.ETag)) + throw new Exception("Failed to get new ETag after update operation."); + + eTag = updateResponse.ETag; + } + } + catch (Exception e) + { + Collector.EmitError("", $"An error occurred while performing a {operation} update to the KeyValueStore", e); + } + return eTag; + } +} diff --git a/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs b/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs new file mode 100644 index 000000000..54a26e7b4 --- /dev/null +++ b/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Documentation.Assembler.Deploying.Serialization; + +public record DescribeKeyValueStoreResponse([property: JsonPropertyName("ETag")] string ETag, [property: JsonPropertyName("KeyValueStore")] KeyValueStore KeyValueStore); +public record KeyValueStore([property: JsonPropertyName("ARN")] string ARN); + +public record ListKeysResponse([property: JsonPropertyName("NextToken")] string? NextToken, [property: JsonPropertyName("Items")] List Items); +public record KeyItem([property: JsonPropertyName("Key")] string Key); + +public record UpdateKeysResponse([property: JsonPropertyName("ETag")] string ETag); + +public record PutKeyRequestListItem +{ + [JsonPropertyName("Key")] + public required string Key { get; init; } + [JsonPropertyName("Value")] + public required string Value { get; init; } +} + +public record DeleteKeyRequestListItem +{ + [JsonPropertyName("Key")] + public required string Key { get; init; } +} + +[JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)] +[JsonSerializable(typeof(DescribeKeyValueStoreResponse))] +[JsonSerializable(typeof(ListKeysResponse))] +[JsonSerializable(typeof(UpdateKeysResponse))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(List))] +internal sealed partial class AwsCloudFrontKeyValueStoreJsonContext : JsonSerializerContext; diff --git a/src/tooling/docs-assembler/docs-assembler.csproj b/src/tooling/docs-assembler/docs-assembler.csproj index 39e4acede..3d445036b 100644 --- a/src/tooling/docs-assembler/docs-assembler.csproj +++ b/src/tooling/docs-assembler/docs-assembler.csproj @@ -17,8 +17,6 @@ - - From 33c6613cef7666555d3419f7d47172338019cbb1 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Tue, 1 Jul 2025 20:13:32 -0300 Subject: [PATCH 2/9] Non-idempotent operations should only be performed once --- .../docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index 0d1f1a394..b93f4d2a5 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -108,7 +108,7 @@ private string ProcessBatchUpdates( AwsCloudFrontKeyValueStoreJsonContext.Default.ListDeleteKeyRequestListItem), _ => string.Empty }; - var responseJson = Capture("aws", "cloudfront-key-value-store", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, + var responseJson = Capture(false, 1, "aws", "cloudfront-key-value-store", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload); var updateResponse = JsonSerializer.Deserialize(responseJson, AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); From ac919372ebc2c2078806abdf737d9e1a9c0aa025 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 09:49:47 -0300 Subject: [PATCH 3/9] Print compact JSON for CLI use --- .../Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs b/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs index 54a26e7b4..7de825bca 100644 --- a/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs +++ b/src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs @@ -28,7 +28,7 @@ public record DeleteKeyRequestListItem public required string Key { get; init; } } -[JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)] +[JsonSourceGenerationOptions(WriteIndented = false, UseStringEnumConverter = true)] [JsonSerializable(typeof(DescribeKeyValueStoreResponse))] [JsonSerializable(typeof(ListKeysResponse))] [JsonSerializable(typeof(UpdateKeysResponse))] From 830bc728167423cafb89761f825064f3b038deae Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 09:50:35 -0300 Subject: [PATCH 4/9] Compact response from AWS CLI, fix arguments according to official docs --- .../Deploying/AwsCloudFrontKeyValueStoreProxy.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index b93f4d2a5..75f005f6e 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -42,7 +42,7 @@ public void UpdateRedirects(string kvsName, IReadOnlyDictionary ConsoleApp.Log("Describing KeyValueStore"); try { - var json = Capture("aws", "cloudfront", "describe-key-value-store", "--name", kvsName); + var json = Capture("aws", "cloudfront", "describe-key-value-store", "--name", kvsName, "|", "jq", "-c"); var describeResponse = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 }) return (describeResponse.KeyValueStore.ARN, describeResponse.ETag); @@ -61,13 +61,13 @@ private HashSet ListAllKeys(string kvsArn) { ConsoleApp.Log("Acquiring existing redirects"); var allKeys = new HashSet(); - string[] baseArgs = ["cloudfront-key-value-store", "list-keys", "--kvs-arn", kvsArn]; + string[] baseArgs = ["cloudfront-keyvaluestore", "list-keys", "--kvs-arn", kvsArn]; string? nextToken = null; try { do { - var json = Capture("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--next-token", nextToken] : []]); + var json = Capture("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--starting-token", nextToken] : [], "|", "jq", "-c"]); var response = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.ListKeysResponse); if (response?.Items != null) @@ -108,8 +108,8 @@ private string ProcessBatchUpdates( AwsCloudFrontKeyValueStoreJsonContext.Default.ListDeleteKeyRequestListItem), _ => string.Empty }; - var responseJson = Capture(false, 1, "aws", "cloudfront-key-value-store", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, - $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload); + var responseJson = Capture(false, 1, "aws", "cloudfront-keyvaluestore", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, + $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload, "|", "jq", "-c"); var updateResponse = JsonSerializer.Deserialize(responseJson, AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); if (string.IsNullOrEmpty(updateResponse?.ETag)) From f1b473c7ba7a44c0a68904d7328dbcc00dba3468 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 10:58:05 -0300 Subject: [PATCH 5/9] Remove pipe, adjust for multiple line output from CLI --- .../ExternalCommands/ExternalCommandExecutor.cs | 17 ++++++++++------- .../AwsCloudFrontKeyValueStoreProxy.cs | 14 +++++++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs index 5c51e9940..2e16f0793 100644 --- a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs +++ b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs @@ -5,7 +5,6 @@ using System.IO.Abstractions; using Elastic.Documentation.Diagnostics; using ProcNet; -using ProcNet.Std; namespace Elastic.Documentation.Tooling.ExternalCommands; @@ -38,11 +37,12 @@ protected void ExecInSilent(Dictionary environmentVars, string b collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); } - protected string[] CaptureMultiple(string binary, params string[] args) + protected string[] CaptureMultiple(string binary, params string[] args) => CaptureMultiple(false, 10, binary, args); + protected string[] CaptureMultiple(bool muteExceptions, int attempts, string binary, params string[] args) { // Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try Exception? e = null; - for (var i = 0; i <= 9; i++) + for (var i = 1; i <= attempts; i++) { try { @@ -55,7 +55,7 @@ protected string[] CaptureMultiple(string binary, params string[] args) } } - if (e is not null) + if (e is not null && !muteExceptions) collector.EmitError("", "failure capturing stdout", e); return []; @@ -70,9 +70,12 @@ string[] CaptureOutput() ConsoleOutWriter = NoopConsoleWriter.Instance }; var result = Proc.Start(arguments); - var output = result.ExitCode != 0 - ? throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}") - : result.ConsoleOut.Select(x => x.Line).ToArray() ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); + + var output = (result.ExitCode, muteExceptions) switch + { + (0, _) or (not 0, true) => result.ConsoleOut.Select(x => x.Line).ToArray() ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"), + (not 0, false) => throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}") + }; return output; } } diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index 75f005f6e..8a49a69b4 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -42,8 +42,8 @@ public void UpdateRedirects(string kvsName, IReadOnlyDictionary ConsoleApp.Log("Describing KeyValueStore"); try { - var json = Capture("aws", "cloudfront", "describe-key-value-store", "--name", kvsName, "|", "jq", "-c"); - var describeResponse = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); + var json = CaptureMultiple("/opt/homebrew/bin/aws-vault", "exec", "elastic-web", "--", "/opt/homebrew/bin/aws", "cloudfront", "describe-key-value-store", "--name", kvsName); + var describeResponse = JsonSerializer.Deserialize(string.Concat(json), AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 }) return (describeResponse.KeyValueStore.ARN, describeResponse.ETag); @@ -67,8 +67,8 @@ private HashSet ListAllKeys(string kvsArn) { do { - var json = Capture("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--starting-token", nextToken] : [], "|", "jq", "-c"]); - var response = JsonSerializer.Deserialize(json, AwsCloudFrontKeyValueStoreJsonContext.Default.ListKeysResponse); + var json = CaptureMultiple("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--starting-token", nextToken] : []]); + var response = JsonSerializer.Deserialize(string.Concat(json), AwsCloudFrontKeyValueStoreJsonContext.Default.ListKeysResponse); if (response?.Items != null) { @@ -108,9 +108,9 @@ private string ProcessBatchUpdates( AwsCloudFrontKeyValueStoreJsonContext.Default.ListDeleteKeyRequestListItem), _ => string.Empty }; - var responseJson = Capture(false, 1, "aws", "cloudfront-keyvaluestore", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, - $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload, "|", "jq", "-c"); - var updateResponse = JsonSerializer.Deserialize(responseJson, AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); + var responseJson = CaptureMultiple(false, 1, "aws", "cloudfront-keyvaluestore", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, + $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload); + var updateResponse = JsonSerializer.Deserialize(string.Concat(responseJson), AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); if (string.IsNullOrEmpty(updateResponse?.ETag)) throw new Exception("Failed to get new ETag after update operation."); From 2248d21c04311a52cb8f6f2e1aeb03d9a88a0d0d Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 10:58:48 -0300 Subject: [PATCH 6/9] Typo --- .../docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index 8a49a69b4..d016a1a43 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -42,7 +42,7 @@ public void UpdateRedirects(string kvsName, IReadOnlyDictionary ConsoleApp.Log("Describing KeyValueStore"); try { - var json = CaptureMultiple("/opt/homebrew/bin/aws-vault", "exec", "elastic-web", "--", "/opt/homebrew/bin/aws", "cloudfront", "describe-key-value-store", "--name", kvsName); + var json = CaptureMultiple("aws", "cloudfront", "describe-key-value-store", "--name", kvsName); var describeResponse = JsonSerializer.Deserialize(string.Concat(json), AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 }) return (describeResponse.KeyValueStore.ARN, describeResponse.ETag); From 8f5cb79b1e1c1803732e6b5e341c2cb876c0ef10 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 11:27:53 -0300 Subject: [PATCH 7/9] Fix update-keys operation --- .../docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index d016a1a43..ac5bcc282 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -109,7 +109,7 @@ private string ProcessBatchUpdates( _ => string.Empty }; var responseJson = CaptureMultiple(false, 1, "aws", "cloudfront-keyvaluestore", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag, - $"--{operation.ToString().ToLowerInvariant()}", "--payload", payload); + $"--{operation.ToString().ToLowerInvariant()}", payload); var updateResponse = JsonSerializer.Deserialize(string.Concat(responseJson), AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse); if (string.IsNullOrEmpty(updateResponse?.ETag)) From f58debb6890f174b3c7257c87e14caa9a4bf9e67 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 14:15:18 -0300 Subject: [PATCH 8/9] Acquire the proper ETag from Cloudfront-KeyValueStore's DescribeKeyValueStore. --- .../AwsCloudFrontKeyValueStoreProxy.cs | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index ac5bcc282..a45187dd4 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -21,8 +21,12 @@ public class AwsCloudFrontKeyValueStoreProxy(DiagnosticsCollector collector, IDi { public void UpdateRedirects(string kvsName, IReadOnlyDictionary sourcedRedirects) { - var (kvsArn, eTag) = DescribeKeyValueStore(kvsName); - if (string.IsNullOrEmpty(kvsArn) || string.IsNullOrEmpty(eTag)) + var kvsArn = DescribeKeyValueStore(kvsName); + if (string.IsNullOrEmpty(kvsArn)) + return; + + var eTag = AcquireETag(kvsArn); + if (string.IsNullOrEmpty(eTag)) return; var existingRedirects = ListAllKeys(kvsArn); @@ -37,23 +41,43 @@ public void UpdateRedirects(string kvsName, IReadOnlyDictionary _ = ProcessBatchUpdates(kvsArn, eTag, toDelete, KvsOperation.Deletes); } - private (string? Arn, string? ETag) DescribeKeyValueStore(string kvsName) + private string DescribeKeyValueStore(string kvsName) { ConsoleApp.Log("Describing KeyValueStore"); try { var json = CaptureMultiple("aws", "cloudfront", "describe-key-value-store", "--name", kvsName); var describeResponse = JsonSerializer.Deserialize(string.Concat(json), AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); - if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 }) - return (describeResponse.KeyValueStore.ARN, describeResponse.ETag); + if (describeResponse?.KeyValueStore is { ARN.Length: > 0 }) + return describeResponse.KeyValueStore.ARN; Collector.EmitError("", "Could not deserialize the DescribeKeyValueStoreResponse"); - return (null, null); + return string.Empty; } catch (Exception e) { Collector.EmitError("", "An error occurred while describing the KeyValueStore", e); - return (null, null); + return string.Empty; + } + } + + private string AcquireETag(string kvsArn) + { + ConsoleApp.Log("Acquiring ETag for updates"); + try + { + var json = CaptureMultiple("aws", "cloudfront-keyvaluestore", "describe-key-value-store", "--kvs-arn", kvsArn); + var describeResponse = JsonSerializer.Deserialize(string.Concat(json), AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse); + if (describeResponse?.ETag is not null) + return describeResponse.ETag; + + Collector.EmitError("", "Could not deserialize Cloudfront-KeyValueStore:DescribeKeyValueStoreResponse"); + return string.Empty; + } + catch (Exception e) + { + Collector.EmitError("", "An error occurred while calling Cloudfront-KeyValueStore:DescribeKeyValueStore", e); + return string.Empty; } } From b23576fe70ed105212619ad504c1dd4fbaa8d137 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Wed, 2 Jul 2025 15:14:59 -0300 Subject: [PATCH 9/9] Add pagination to list-keys command --- .../docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs index a45187dd4..8e77585f9 100644 --- a/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs +++ b/src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs @@ -85,7 +85,7 @@ private HashSet ListAllKeys(string kvsArn) { ConsoleApp.Log("Acquiring existing redirects"); var allKeys = new HashSet(); - string[] baseArgs = ["cloudfront-keyvaluestore", "list-keys", "--kvs-arn", kvsArn]; + string[] baseArgs = ["cloudfront-keyvaluestore", "list-keys", "--kvs-arn", kvsArn, "--page-size", "50", "--max-items", "50"]; string? nextToken = null; try {