From ac7af0d2eaaa05f438ec32d86d3f0432b4c4f974 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 31 Oct 2023 15:59:25 +0000 Subject: [PATCH 1/3] shortening s3 policy Signed-off-by: Neil South --- src/TaskManager/TaskManager/TaskManager.cs | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/TaskManager/TaskManager/TaskManager.cs b/src/TaskManager/TaskManager/TaskManager.cs index 80369aa62..3502ab491 100644 --- a/src/TaskManager/TaskManager/TaskManager.cs +++ b/src/TaskManager/TaskManager/TaskManager.cs @@ -447,7 +447,7 @@ private async Task HandleDispatchTask(JsonMessage message) try { - if (PluginStrings.PlugsRequiresPermanentAccoutns.Contains( + if (PluginStrings.PlugsRequiresPermanentAccounts.Contains( message.Body.TaskPluginType, StringComparer.InvariantCultureIgnoreCase)) { @@ -559,7 +559,13 @@ private async Task PopulateTemporaryStorageCredentials(params Messaging.Common.S foreach (var storage in storages) { - var credentials = await _storageService.CreateTemporaryCredentialsAsync(storage.Bucket, storage.RelativeRootPath, _options.Value.TaskManager.TemporaryStorageCredentialDurationSeconds, _cancellationToken).ConfigureAwait(false); + var credentials = await _storageService.CreateTemporaryCredentialsAsync( + storage.Bucket, + ShortenStoragePath(storage.RelativeRootPath), + _options.Value.TaskManager.TemporaryStorageCredentialDurationSeconds, + _cancellationToken) + .ConfigureAwait(false); + storage.Credentials = new Credentials { AccessKey = credentials.AccessKeyId, @@ -569,6 +575,19 @@ private async Task PopulateTemporaryStorageCredentials(params Messaging.Common.S } } + // added because AWS s3 policy creation is by defualt limited to 2048 characters, which + // can easily be surpassed with long multipart path names. + private string ShortenStoragePath(string path) + { + var pathParts = path.Split('/'); + if (pathParts.Length <= 3) + { + return path; + } + + return $"{pathParts[0]}/{pathParts[1]}/{pathParts[2]}"; + } + private void AcknowledgeMessage(JsonMessage message) { Guard.Against.NullService(_messageBrokerSubscriberService, nameof(IMessageBrokerSubscriberService)); From 1b0fcce614d6d4f99552c08c595aca285ce0ff30 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 31 Oct 2023 16:08:33 +0000 Subject: [PATCH 2/3] fix typo Signed-off-by: Neil South --- src/TaskManager/TaskManager/PluginStrings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TaskManager/TaskManager/PluginStrings.cs b/src/TaskManager/TaskManager/PluginStrings.cs index 6a8c5156a..3d8ba6f6d 100644 --- a/src/TaskManager/TaskManager/PluginStrings.cs +++ b/src/TaskManager/TaskManager/PluginStrings.cs @@ -24,7 +24,7 @@ public static class PluginStrings public const string Docker = "docker"; - public static readonly IReadOnlyList PlugsRequiresPermanentAccoutns = new List() { Argo, Docker }; + public static readonly IReadOnlyList PlugsRequiresPermanentAccounts = new List() { Argo, Docker }; } } #pragma warning restore SA1600 // Elements should be documented From 82042a31dc0036039e028ef8f31311e2d61fc6fa Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 31 Oct 2023 16:44:15 +0000 Subject: [PATCH 3/3] fixing up paths Signed-off-by: Neil South --- src/TaskManager/TaskManager/TaskManager.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/TaskManager/TaskManager/TaskManager.cs b/src/TaskManager/TaskManager/TaskManager.cs index 3502ab491..a94c6d069 100644 --- a/src/TaskManager/TaskManager/TaskManager.cs +++ b/src/TaskManager/TaskManager/TaskManager.cs @@ -579,13 +579,14 @@ private async Task PopulateTemporaryStorageCredentials(params Messaging.Common.S // can easily be surpassed with long multipart path names. private string ShortenStoragePath(string path) { - var pathParts = path.Split('/'); + var pathParts = path.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); if (pathParts.Length <= 3) { return path; } - return $"{pathParts[0]}/{pathParts[1]}/{pathParts[2]}"; + var startsWith = path[0] == '/' ? "/" : string.Empty; + return $"{startsWith}{pathParts[0]}/{pathParts[1]}/{pathParts[2]}"; } private void AcknowledgeMessage(JsonMessage message)