Ai 275

src/TaskManager/TaskManager/appsettings.json

@@ -108,7 +108,8 @@
         "deadLetterExchange": "monaideploy-dead-letter",
         "exportRequestQueue": "export_tasks",
         "deliveryLimit": 3,
-        "requeueDelay": 30
+        "requeueDelay": 3,
+        "prefetchCount": "5"
       }
     },
     "storage": {

src/WorkflowManager/Common/Services/PayloadService.cs

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-using Ardalis.GuardClauses;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Monai.Deploy.Messaging.Events;
@@ -83,6 +82,7 @@ public PayloadService(
                 }
 
                 var patientDetails = await _dicomService.GetPayloadPatientDetailsAsync(eventPayload.PayloadId.ToString(), eventPayload.Bucket);
+                var dict = await _dicomService.GetMetaData(eventPayload.PayloadId.ToString(), eventPayload.Bucket).ConfigureAwait(false);
 
                 var payload = new Payload
                 {
@@ -96,7 +96,8 @@ public PayloadService(
                     Timestamp = eventPayload.Timestamp,
                     PatientDetails = patientDetails,
                     PayloadDeleted = PayloadDeleted.No,
-                    Expires = await GetExpiry(DateTime.UtcNow, eventPayload.WorkflowInstanceId)
+                    Expires = await GetExpiry(DateTime.UtcNow, eventPayload.WorkflowInstanceId),
+                    SeriesInstanceUid = _dicomService.GetSeriesInstanceUID(dict)
                 };
 
                 if (await _payloadRepository.CreateAsync(payload))
@@ -197,13 +198,7 @@ public async Task<bool> DeletePayloadFromStorageAsync(string payloadId)
         {
             ArgumentNullException.ThrowIfNullOrWhiteSpace(payloadId, nameof(payloadId));
 
-            var payload = await GetByIdAsync(payloadId);
-
-            if (payload is null)
-            {
-                throw new MonaiNotFoundException($"Payload with ID: {payloadId} not found");
-            }
-
+            var payload = await GetByIdAsync(payloadId) ?? throw new MonaiNotFoundException($"Payload with ID: {payloadId} not found");
             if (payload.PayloadDeleted == PayloadDeleted.InProgress || payload.PayloadDeleted == PayloadDeleted.Yes)
             {
                 throw new MonaiBadRequestException($"Deletion of files for payload ID: {payloadId} already in progress or already deleted");

src/WorkflowManager/Contracts/Migrations/M001_Payload_addVerion.cs

@@ -21,7 +21,7 @@ namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
 {
     public class M001_Payload_addVerion : DocumentMigration<Payload>
     {
-        public M001_Payload_addVerion() : base("1.0.0") { }
+        public M001_Payload_addVerion() : base("1.0.1") { }
 
         public override void Up(BsonDocument document)
         {

src/WorkflowManager/Contracts/Migrations/M002_Payload_addPayloadDeleted.cs

@@ -21,7 +21,7 @@ namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
 {
     public class M002_Payload_addPayloadDeleted : DocumentMigration<Payload>
     {
-        public M002_Payload_addPayloadDeleted() : base("1.0.1") { }
+        public M002_Payload_addPayloadDeleted() : base("1.0.2") { }
 
         public override void Up(BsonDocument document)
         {

src/WorkflowManager/Contracts/Migrations/M005_Payload_seriesUid.cs

@@ -0,0 +1,42 @@
+// 
+// Copyright 2023 Guy’s and St Thomas’ NHS Foundation Trust
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Monai.Deploy.WorkflowManager.Common.Contracts.Models;
+using Mongo.Migration.Migrations.Document;
+using MongoDB.Bson;
+
+namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
+{
+    public class M005_Payload_seriesUid : DocumentMigration<Payload>
+    {
+        public M005_Payload_seriesUid() : base("1.0.5") { }
+
+        public override void Up(BsonDocument document)
+        {
+            document.Add("SeriesInstanceUid", BsonNull.Create(null).ToJson(), true);
+        }
+
+        public override void Down(BsonDocument document)
+        {
+            try
+            {
+                document.Remove("SeriesInstanceUid");
+            }
+            catch
+            {  // can ignore we dont want failures stopping startup !
+            }
+        }
+    }
+}

src/WorkflowManager/Contracts/Models/Payload.cs

@@ -27,11 +27,11 @@
 
 namespace Monai.Deploy.WorkflowManager.Common.Contracts.Models
 {
-    [CollectionLocation("Payloads"), RuntimeVersion("1.0.4")]
+    [CollectionLocation("Payloads"), RuntimeVersion("1.0.5")]
     public class Payload : IDocument
     {
         [JsonConverter(typeof(DocumentVersionConvert)), BsonSerializer(typeof(DocumentVersionConverBson))]
-        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 4);
+        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 5);
 
         [JsonProperty(PropertyName = "id")]
         public string Id { get; set; } = string.Empty;
@@ -71,6 +71,9 @@ public class Payload : IDocument
         [JsonProperty(PropertyName = "expires")]
         public DateTime? Expires { get; set; }
 
+        [JsonProperty(PropertyName = "series_instance_uid")]
+        public string? SeriesInstanceUid { get; set; }
+
     }
 
     public enum PayloadDeleted

src/WorkflowManager/Contracts/Models/PayloadDto.cs

@@ -36,6 +36,7 @@ public PayloadDto(Payload payload)
             Files = payload.Files;
             PatientDetails = payload.PatientDetails;
             PayloadDeleted = payload.PayloadDeleted;
+            SeriesInstanceUid = payload.SeriesInstanceUid;
         }
 
         [JsonProperty(PropertyName = "payload_status")]

src/WorkflowManager/Logging/Log.200000.Workflow.cs

@@ -111,5 +111,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 210018, Level = LogLevel.Error, Message = "ExportList or Artifacts are empty! workflowInstanceId {workflowInstanceId} TaskId {taskId}")]
         public static partial void ExportListOrArtifactsAreEmpty(this ILogger logger, string taskId, string workflowInstanceId);
+
+        [LoggerMessage(EventId = 210019, Level = LogLevel.Error, Message = "Task is missing required input artifacts {taskId} Artifacts {ArtifactsJson}")]
+        public static partial void TaskIsMissingRequiredInputArtifacts(this ILogger logger, string taskId, string ArtifactsJson);
     }
 }

src/WorkflowManager/Logging/Log.600000.Dicom.cs

@@ -40,5 +40,11 @@ public static partial class Log
 
         [LoggerMessage(EventId = 600006, Level = LogLevel.Debug, Message = "Dicom export marked as failed with {fileStatusCount} files marked as exported.")]
         public static partial void DicomExportFailed(this ILogger logger, string fileStatusCount);
+
+        [LoggerMessage(EventId = 600007, Level = LogLevel.Error, Message = "Failed to get DICOM metadata from bucket {bucketId}. Payload: {payloadId}")]
+        public static partial void FailedToGetDicomMetadataFromBucket(this ILogger logger, string payloadId, string bucketId, Exception ex);
+
+        [LoggerMessage(EventId = 600008, Level = LogLevel.Error, Message = "Failed to get DICOM tag {dicomTag} from dictionary")]
+        public static partial void FailedToGetDicomTagFromDictoionary(this ILogger logger, string dicomTag, Exception ex);
     }
 }

src/WorkflowManager/Logging/Log.700000.Artifact.cs

@@ -57,6 +57,15 @@ public static partial class Log
         [LoggerMessage(EventId = 700011, Level = LogLevel.Debug, Message = "adding files to workflowInstance {workflowInstanceId} :Task {taskId} :  {artifactList}")]
         public static partial void AddingFilesToWorkflowInstance(this ILogger logger, string workflowInstanceId, string taskId, string artifactList);
 
+        [LoggerMessage(EventId = 700012, Level = LogLevel.Error, Message = "Error finding Task :{taskId}")]
+        public static partial void ErrorFindingTask(this ILogger logger, string taskId);
+
+        [LoggerMessage(EventId = 700013, Level = LogLevel.Error, Message = "Error finding Task :{taskId} or previousTask {previousTask}")]
+        public static partial void ErrorFindingTaskOrPrevious(this ILogger logger, string taskId, string previousTask);
+
+        [LoggerMessage(EventId = 700014, Level = LogLevel.Warning, Message = "Error Task :{taskId} cant be trigger as it has missing artifacts {missingtypesJson}")]
+        public static partial void ErrorTaskMissingArtifacts(this ILogger logger, string taskId, string missingtypesJson);
+
 
     }
 }

src/WorkflowManager/Storage/Constants/DicomTagConstants.cs

@@ -29,5 +29,7 @@ public static class DicomTagConstants
         public const string PatientAgeTag = "00101010";
 
         public const string PatientHospitalIdTag = "00100021";
+
+        public const string SeriesInstanceUIDTag = "0020000E";
     }
 }

src/WorkflowManager/Storage/Services/DicomService.cs

@@ -16,7 +16,6 @@
 
 using System.Globalization;
 using System.Text;
-using Ardalis.GuardClauses;
 using Microsoft.Extensions.Logging;
 using Monai.Deploy.Storage.API;
 using Monai.Deploy.WorkflowManager.Common.Contracts.Models;
@@ -69,18 +68,18 @@ public async Task<PatientDetails> GetPayloadPatientDetailsAsync(string payloadId
             ArgumentNullException.ThrowIfNullOrWhiteSpace(bucketName, nameof(bucketName));
             ArgumentNullException.ThrowIfNullOrWhiteSpace(payloadId, nameof(payloadId));
 
-            var items = await _storageService.ListObjectsAsync(bucketName, $"{payloadId}/dcm", true);
+            var dict = await GetMetaData(payloadId, bucketName);
 
             var patientDetails = new PatientDetails
             {
-                PatientName = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientNameTag),
-                PatientId = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientIdTag),
-                PatientSex = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientSexTag),
-                PatientAge = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientAgeTag),
-                PatientHospitalId = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientHospitalIdTag)
+                PatientName = GetFirstValueAsync(dict, DicomTagConstants.PatientNameTag),
+                PatientId = GetFirstValueAsync(dict, DicomTagConstants.PatientIdTag),
+                PatientSex = GetFirstValueAsync(dict, DicomTagConstants.PatientSexTag),
+                PatientAge = GetFirstValueAsync(dict, DicomTagConstants.PatientAgeTag),
+                PatientHospitalId = GetFirstValueAsync(dict, DicomTagConstants.PatientHospitalIdTag)
             };
 
-            var dob = await GetFirstValueAsync(items, payloadId, bucketName, DicomTagConstants.PatientDateOfBirthTag);
+            var dob = GetFirstValueAsync(dict, DicomTagConstants.PatientDateOfBirthTag);
 
             if (DateTime.TryParseExact(dob, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateOfBirth))
             {
@@ -90,12 +89,38 @@ public async Task<PatientDetails> GetPayloadPatientDetailsAsync(string payloadId
             return patientDetails;
         }
 
-        public async Task<string?> GetFirstValueAsync(IList<VirtualFileInfo> items, string payloadId, string bucketId, string keyId)
+        private string? GetFirstValueAsync(Dictionary<string, DicomValue>? dict, string keyId)
         {
-            ArgumentNullException.ThrowIfNullOrWhiteSpace(bucketId, nameof(bucketId));
-            ArgumentNullException.ThrowIfNullOrWhiteSpace(payloadId, nameof(payloadId));
             ArgumentNullException.ThrowIfNullOrWhiteSpace(keyId, nameof(keyId));
+            if (dict is null)
+            {
+                return null;
+            }
 
+            try
+            {
+                var value = GetValue(dict, keyId);
+
+                if (!string.IsNullOrWhiteSpace(value))
+                {
+                    return value;
+                }
+                return null;
+            }
+            catch (Exception e)
+            {
+                _logger.FailedToGetDicomTagFromDictoionary(keyId, e);
+            }
+
+            return null;
+        }
+
+        public async Task<Dictionary<string, DicomValue>?> GetMetaData(string payloadId, string bucketId)
+        {
+            ArgumentNullException.ThrowIfNullOrWhiteSpace(bucketId, nameof(bucketId));
+            ArgumentNullException.ThrowIfNullOrWhiteSpace(payloadId, nameof(payloadId));
+            var items = await _storageService.ListObjectsAsync(bucketId, $"{payloadId}/dcm", true);
+            var dict = new Dictionary<string, DicomValue>(StringComparer.OrdinalIgnoreCase);
             try
             {
                 if (items is null || items.Any() is false)
@@ -113,20 +138,21 @@ public async Task<PatientDetails> GetPayloadPatientDetailsAsync(string payloadId
                     var stream = await _storageService.GetObjectAsync(bucketId, filePath);
                     var jsonStr = Encoding.UTF8.GetString(((MemoryStream)stream).ToArray());
 
-                    var dict = new Dictionary<string, DicomValue>(StringComparer.OrdinalIgnoreCase);
-                    JsonConvert.PopulateObject(jsonStr, dict);
+                    var dictCurrent = new Dictionary<string, DicomValue>(StringComparer.OrdinalIgnoreCase);
+                    JsonConvert.PopulateObject(jsonStr, dictCurrent);
 
-                    var value = GetValue(dict, keyId);
 
-                    if (!string.IsNullOrWhiteSpace(value))
+                    // merge the two dictionaries
+                    foreach (var (key, value) in dictCurrent)
                     {
-                        return value;
+                        dict.TryAdd(key, value);
                     }
                 }
+                return dict;
             }
             catch (Exception e)
             {
-                _logger.FailedToGetDicomTagFromPayload(payloadId, keyId, bucketId, e);
+                _logger.FailedToGetDicomMetadataFromBucket(payloadId, bucketId, e);
             }
 
             return null;
@@ -141,7 +167,7 @@ public async Task<IEnumerable<string>> GetDicomPathsForTaskAsync(string outputDi
 
             var dicomFiles = files?.Where(f => f.FilePath.EndsWith(".dcm"));
 
-            return dicomFiles?.Select(d => d.FilePath)?.ToList() ?? new List<string>();
+            return dicomFiles?.Select(d => d.FilePath) ?? [];
         }
 
         public async Task<string> GetAnyValueAsync(string keyId, string payloadId, string bucketId)
@@ -180,7 +206,7 @@ public async Task<string> GetAllValueAsync(string keyId, string payloadId, strin
             var matchValue = await GetDcmJsonFileValueAtIndexAsync(0, path, bucketId, keyId, listOfJsonFiles);
             var fileCount = listOfJsonFiles.Count;
 
-            for (int i = 0; i < fileCount; i++)
+            for (var i = 0; i < fileCount; i++)
             {
                 if (listOfJsonFiles[i].Filename.EndsWith(".dcm"))
                 {
@@ -229,11 +255,6 @@ public async Task<string> GetDcmJsonFileValueAtIndexAsync(int index,
 
         public string GetValue(Dictionary<string, DicomValue> dict, string keyId)
         {
-            if (dict.Any() is false)
-            {
-                return string.Empty;
-            }
-
             var result = string.Empty;
 
             if (dict.TryGetValue(keyId, out var value))
@@ -261,6 +282,20 @@ public string GetValue(Dictionary<string, DicomValue> dict, string keyId)
             return result;
         }
 
+        public string? GetSeriesInstanceUID(Dictionary<string, DicomValue>? dict)
+        {
+            if (dict is null)
+            {
+                return null;
+            }
+
+            if (dict.TryGetValue(DicomTagConstants.SeriesInstanceUIDTag, out var value))
+            {
+                return value.Value.ToString();
+            }
+            return null;
+        }
+
         private string TryGetValueAndLogSupported(string vrFullString, DicomValue value, string jsonString)
         {
             var result = TryGetValue(value);

src/WorkflowManager/Storage/Services/IDicomService.cs

@@ -63,5 +63,20 @@ public interface IDicomService
         /// <param name="keyId"></param>
         /// <returns></returns>
         string GetValue(Dictionary<string, DicomValue> dict, string keyId);
+
+        /// <summary>
+        /// Gets the first metadata froma payloads folder.
+        /// </summary>
+        /// <param name="payloadId"></param>
+        /// <param name="bucketId"></param>
+        /// <returns>a dictionary of tags and values</returns>
+        Task<Dictionary<string, DicomValue>?> GetMetaData(string payloadId, string bucketId);
+
+        /// <summary>
+        /// Get the seriers instance UID from the metadata.
+        /// </summary>
+        /// <param name="dict"></param>
+        /// <returns>a string containing the seriers instanceUid</returns>
+        string? GetSeriesInstanceUID(Dictionary<string, DicomValue>? dict);
     }
 }