From f15188ee35b7f89be3be453ed2f3e951b4614768 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Wed, 21 Sep 2022 14:32:40 +0100 Subject: [PATCH 1/4] artifact logging Signed-off-by: Jack Schofield --- src/WorkflowManager/Logging/Logging/Log.cs | 13 +++++++++++++ .../WorkflowExecuter/Common/ArtifactMapper.cs | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/src/WorkflowManager/Logging/Logging/Log.cs b/src/WorkflowManager/Logging/Logging/Log.cs index da6f22fab..4a77eed52 100644 --- a/src/WorkflowManager/Logging/Logging/Log.cs +++ b/src/WorkflowManager/Logging/Logging/Log.cs @@ -152,6 +152,19 @@ public static void LogControllerEndTime(this ILogger logger, ResultExecutedConte [LoggerMessage(EventId = 33, Level = LogLevel.Debug, Message = "Task destination condition for task {taskId} with condition: {conditions} resolved to false.")] public static partial void TaskDestinationConditionFalse(this ILogger logger, string conditions, string taskId); + public static void LogArtifactPassing(this ILogger logger, Artifact artifact, string path, string artifactType, bool exists) + { + var objectLog = new ObjectLog + { + Message = "Artifact Passed", + Object = LoggerHelpers.ToLogArtifactPassingObject(artifact, path, artifactType, exists) + }; + + var jsonString = JsonConvert.SerializeObject(objectLog); + + logger.LogInformation(34, message: jsonString); + } + [LoggerMessage(EventId = 34, Level = LogLevel.Debug, Message = "Payload already exists for {payloadId}. This is likley due to being requeued")] public static partial void PayloadAlreadyExists(this ILogger logger, string payloadId); } diff --git a/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs index 294d90b46..4d0f6e40f 100644 --- a/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs +++ b/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs @@ -70,9 +70,13 @@ public async Task> ConvertArtifactVariablesToPath(Art { artifactPathDictionary.Add(mappedArtifact.Key, mappedArtifact.Value); + _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Output", true); + continue; } + _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Output", false); + if (artifact.Mandatory) { throw new FileNotFoundException($"Mandatory artifact was not found: {artifact.Name}, {artifact.Value}"); From df0dd98d9af57532d5d7a6209f8bbd50c2e5a67c Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Wed, 21 Sep 2022 15:02:52 +0100 Subject: [PATCH 2/4] fix duplicate mapping Signed-off-by: Jack Schofield --- .../WorkflowExecuter/Services/WorkflowExecuterService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs index eb5032a2d..eade65103 100644 --- a/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -119,8 +119,8 @@ public async Task ProcessPayload(WorkflowRequestEvent message, Payload pay var workflowInstances = new List(); var tasks = workflows.Select(workflow => CreateWorkflowInstanceAsync(message, workflow)); - await Task.WhenAll(tasks).ConfigureAwait(false); - workflowInstances.AddRange(tasks.Select(t => t.Result)); + var newInstances = await Task.WhenAll(tasks).ConfigureAwait(false); + workflowInstances.AddRange(newInstances); var existingInstances = await _workflowInstanceRepository.GetByWorkflowsIdsAsync(workflowInstances.Select(w => w.WorkflowId).ToList()); From 2cb35a23bd7452bb8cb48aa419b7adf9e8d545f8 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Thu, 22 Sep 2022 15:02:19 +0100 Subject: [PATCH 3/4] add output logging Signed-off-by: Jack Schofield --- .../WorkflowExecuter/Common/ArtifactMapper.cs | 4 ++-- .../Services/WorkflowExecuterService.cs | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs index 4d0f6e40f..bdba54fa4 100644 --- a/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs +++ b/src/WorkflowManager/WorkflowExecuter/Common/ArtifactMapper.cs @@ -70,12 +70,12 @@ public async Task> ConvertArtifactVariablesToPath(Art { artifactPathDictionary.Add(mappedArtifact.Key, mappedArtifact.Value); - _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Output", true); + _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Pre-Task Output Path Mapping", true); continue; } - _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Output", false); + _logger.LogArtifactPassing(artifact, mappedArtifact.Value, shouldExistYet ? "Input" : "Pre-Task Output Path Mapping", false); if (artifact.Mandatory) { diff --git a/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs index eade65103..69c48d63b 100644 --- a/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -423,13 +423,17 @@ private async Task HandleOutputArtifacts(WorkflowInstance workflowInstance var validOutputArtifacts = await _storageService.VerifyObjectsExistAsync(workflowInstance.BucketId, artifactDict); - workflowInstance.Tasks?.ForEach(t => + foreach (var artifact in artifactDict) { - if (t.TaskId == task.TaskId) - { - t.OutputArtifacts = validOutputArtifacts; - } - }); + _logger.LogArtifactPassing(new Artifact { Name = artifact.Key, Value = artifact.Value }, artifact.Value, "Post-Task Output Artifact", validOutputArtifacts.ContainsKey(artifact.Key)); + } + + var currentTask = workflowInstance.Tasks?.FirstOrDefault(t => t.TaskId == task.TaskId); + + if (currentTask is not null) + { + currentTask.OutputArtifacts = validOutputArtifacts; + } if (validOutputArtifacts is not null && validOutputArtifacts.Any()) { From 3889e104708a1987abee10360a0d2eb3993cafcc Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Wed, 28 Sep 2022 09:23:59 +0100 Subject: [PATCH 4/4] use new test object format Signed-off-by: Jack Schofield --- src/WorkflowManager/Logging/Logging/Log.cs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/WorkflowManager/Logging/Logging/Log.cs b/src/WorkflowManager/Logging/Logging/Log.cs index 4a77eed52..4018210b0 100644 --- a/src/WorkflowManager/Logging/Logging/Log.cs +++ b/src/WorkflowManager/Logging/Logging/Log.cs @@ -14,6 +14,7 @@ * limitations under the License. */ +using Amazon.Runtime.Internal; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.Extensions.Logging; @@ -154,18 +155,11 @@ public static void LogControllerEndTime(this ILogger logger, ResultExecutedConte public static void LogArtifactPassing(this ILogger logger, Artifact artifact, string path, string artifactType, bool exists) { - var objectLog = new ObjectLog - { - Message = "Artifact Passed", - Object = LoggerHelpers.ToLogArtifactPassingObject(artifact, path, artifactType, exists) - }; - - var jsonString = JsonConvert.SerializeObject(objectLog); - - logger.LogInformation(34, message: jsonString); + logger.LogInformation(34, "Artifact Passed data Artifact {artifact}, Path {path}, ArtifactType {artifactType}, Exists {exists}", + JsonConvert.SerializeObject(artifact), path, artifactType, exists); } - [LoggerMessage(EventId = 34, Level = LogLevel.Debug, Message = "Payload already exists for {payloadId}. This is likley due to being requeued")] + [LoggerMessage(EventId = 35, Level = LogLevel.Debug, Message = "Payload already exists for {payloadId}. This is likley due to being requeued")] public static partial void PayloadAlreadyExists(this ILogger logger, string payloadId); } }