Tests for exporting a folder

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Features/TaskStatusUpdate.feature

@@ -86,11 +86,15 @@ Scenario Outline: Publish an valid Task Update event with a status that is inval
     | WFI_Task_Status_Canceled  | Task_Status_Update_Status_Invalid_When_Canceled  | Accepted         |
 
 @TaskExport
-Scenario: Export task with single destination is in progress, export message is sent 
-    Given I have a clinical workflow Workflow_Revision_for_export_single_dest_1
-    And I have a Workflow Instance Workflow_Instance_for_export_single_dest_1 with no artifacts
-    When I publish a Task Update Message Task_status_update_for_export_single_dest_1 with artifacts output_metadata in minio
+Scenario Outline: Export task with single destination is in progress, export message is sent 
+    Given I have a clinical workflow <workflow>
+    And I have a Workflow Instance <workflowInstance> with no artifacts
+    When I publish a Task Update Message <taskUpdate> with artifacts <minioFiles> in minio
     Then 1 Export Request message is published
+    Examples:
+    | workflow                                   | workflowInstance                           | taskUpdate                                  | minioFiles                     |
+    | Workflow_Revision_for_export_single_dest_1 | Workflow_Instance_for_export_single_dest_1 | Task_status_update_for_export_single_dest_1 | output_metadata                |
+    | Workflow_Revision_for_export_folder        | Workflow_Instance_for_export_folder        | Task_status_update_for_export_folder        | output_metadata_multiple_files |
 
 @TaskExport
 Scenario: Export task with mutliple destinations is in progress, export message is sent 

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Models/Storage/VirtuaFileInfo.cs

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2022 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Ardalis.GuardClauses;
+
+namespace Monai.Deploy.WorkflowManager.WorkflowExecutor.IntegrationTests.Models.Storage
+{
+    /// <summary>
+    /// Represents a file stored on the virtual storage device.
+    /// </summary>
+    public class VirtualFileInfo
+    {
+        /// <summary>
+        /// Gets or set the name of the file
+        /// </summary>
+        public string Filename { get; set; }
+
+        /// <summary>
+        /// Gets or sets the (non-rooted) path of the file
+        /// </summary>
+        public string FilePath { get; set; }
+
+        /// <summary>
+        /// Gets or set the etag of the file
+        /// </summary>
+        public string ETag { get; set; }
+
+        /// <summary>
+        /// Gets or sets the size of the file
+        /// </summary>
+        public ulong Size { get; set; }
+
+        /// <summary>
+        /// Gets or set the last modified date time of the file
+        /// </summary>
+        public DateTime? LastModifiedDateTime { get; set; }
+
+        /// <summary>
+        /// Gets or sets the metadata associated with the file
+        /// </summary>
+        public Dictionary<string, string> Metadata { get; set; }
+
+        public VirtualFileInfo(string filename, string filePath, string etag, ulong size)
+        {
+            Guard.Against.Null(filename, nameof(filename));
+            Guard.Against.Null(filePath, nameof(filePath));
+            Guard.Against.Null(etag, nameof(etag));
+
+            Filename = filename;
+            FilePath = filePath;
+            ETag = etag;
+            Size = size;
+
+            Metadata = new Dictionary<string, string>();
+        }
+    }
+}

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Monai.Deploy.WorkflowManager.WorkflowExecutor.IntegrationTests.csproj

@@ -79,6 +79,15 @@
     <None Update="DICOMs\null_patient_metadata\dcm\07051db3-3c1d-4bf2-8764-ba45dc918e74.dcm.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="DICOMs\output_metadata_multiple_files\dcm\output3.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="DICOMs\output_metadata_multiple_files\dcm\output2.dcm">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="DICOMs\output_metadata_multiple_files\dcm\output1.dcm">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="DICOMs\output_metadata\dcm\output.dcm">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/StepDefinitions/TaskStatusUpdateStepDefinitions.cs

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+using System.Threading.Tasks;
 using BoDi;
 using Monai.Deploy.Messaging.Events;
 using Monai.Deploy.Messaging.Messages;

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/Assertions.cs

@@ -225,7 +225,16 @@ public void AssertExportRequestEvent(ExportRequestEvent exportRequestEvent, Work
             }
 
             workflowRevisionTask.ExportDestinations.Select(x => x.Name).ToArray().Should().BeEquivalentTo(exportRequestEvent.Destinations);
-            exportRequestEvent.Files.Count().Should().Be(workflowRevisionTask.Artifacts.Input.Count());
+            if (taskUpdateEvent.Outputs[0].RelativeRootPath.EndsWith(".dcm"))
+            {
+                exportRequestEvent.Files.Count().Should().Be(workflowRevisionTask.Artifacts.Input.Count());
+            }
+            else
+            {
+                var filesList = MinioClient.ListFilesFromDir(TestExecutionConfig.MinioConfig.Bucket, taskUpdateEvent.Outputs[0].RelativeRootPath).Result;
+                var filteredFileList = filesList.Where(f => f.FilePath.EndsWith(".dcm"));
+                exportRequestEvent.Files.Count().Should().Be(filteredFileList.Count());
+            }
             exportRequestEvent.WorkflowInstanceId.Should().Match(workflowInstance.Id);
             exportRequestEvent.ExportTaskId.Should().Match(workflowInstanceTask?.TaskId);
         }

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/MinioClientUtil.cs

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
+using System.Reactive.Linq;
 using Minio;
+using Monai.Deploy.Storage.API;
 using Monai.Deploy.WorkflowManager.IntegrationTests.POCO;
 using Polly;
 using Polly.Retry;
-using System.Reactive.Linq;
-
+using File = System.IO.File;
 
 namespace Monai.Deploy.WorkflowManager.IntegrationTests.Support
 {
@@ -180,5 +181,38 @@ public async Task RemoveObjects(string bucketName, string objectName)
                 await Client.RemoveObjectAsync(bucketName, objectName);
             }
         }
+
+        public async Task<IList<VirtualFileInfo>> ListFilesFromDir(string bucketName, string objectName)
+        {
+            var cancellationToken = default(CancellationToken);
+            return await Task.Run(() =>
+            {
+                var files = new List<VirtualFileInfo>();
+                var listArgs = new ListObjectsArgs()
+                    .WithBucket(bucketName)
+                    .WithPrefix(objectName)
+                    .WithRecursive(true);
+
+                var objservable = Client.ListObjectsAsync(listArgs, cancellationToken);
+                var completedEvent = new ManualResetEventSlim(false);
+                objservable.Subscribe(item =>
+                {
+                    if (!item.IsDir)
+                    {
+                        files.Add(new VirtualFileInfo(Path.GetFileName(item.Key), item.Key, item.ETag, item.Size)
+                        {
+                            LastModifiedDateTime = item.LastModifiedDateTime
+                        });
+                    }
+                },
+                error =>
+                {
+                },
+                () => completedEvent.Set(), cancellationToken);
+
+                completedEvent.Wait(cancellationToken);
+                return files;
+            }).ConfigureAwait(false);
+        }
     }
 }

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/TestData/TaskUpdateTestData.cs

@@ -838,6 +838,37 @@ public static TaskUpdateEvent CreateTaskUpdateEvent(string workflowInstanceName)
                 }
             },
             new TaskUpdateTestData()
+            {
+                Name = "Task_status_update_for_export_folder",
+                TaskUpdateEvent = new TaskUpdateEvent()
+                {
+                    WorkflowInstanceId = Helper.GetWorkflowInstanceByName("Workflow_Instance_for_export_folder").WorkflowInstance.Id,
+                    ExecutionId = Helper.GetWorkflowInstanceByName("Workflow_Instance_for_export_folder").WorkflowInstance.Tasks[0].ExecutionId,
+                    CorrelationId = Guid.NewGuid().ToString(),
+                    Reason = FailureReason.None,
+                    Message = "Task Message",
+                    TaskId = Helper.GetWorkflowInstanceByName("Workflow_Instance_for_export_folder").WorkflowInstance.Tasks[0].TaskId,
+                    Outputs = new List<Messaging.Common.Storage>
+                    {
+                        new Messaging.Common.Storage()
+                        {
+                            Name = "output",
+                            Endpoint = "//output.dcm",
+                            Credentials = new Messaging.Common.Credentials()
+                            {
+                                AccessKey = "test1",
+                                AccessToken = "test",
+                            },
+                            Bucket = "bucket1",
+                            RelativeRootPath = GetRelativePathForOutputArtifacts("Workflow_Instance_for_export_folder")
+                        }
+                    },
+                    Metadata = new Dictionary<string, object>()
+                    {
+                    }
+                }
+            },
+            new TaskUpdateTestData()
             {
                 Name = "Task_status_update_for_export_multi_dest_1",
                 TaskUpdateEvent = new TaskUpdateEvent()

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/TestData/WorkflowInstanceTestData.cs

@@ -16,6 +16,7 @@
 
 using Monai.Deploy.Messaging.Events;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
+using Monai.Deploy.WorkflowManager.Controllers;
 using Monai.Deploy.WorkflowManager.IntegrationTests.POCO;
 #pragma warning disable CS8602 // Dereference of a possibly null reference.
 #pragma warning disable CS8601 // Possible null reference assignment.
@@ -1230,6 +1231,37 @@ public static WorkflowInstance CreateWorkflowInstance(string workflowName)
                 }
             },
             new WorkflowInstanceTestData()
+            {
+                Name = "Workflow_Instance_for_export_folder",
+                WorkflowInstance = new WorkflowInstance()
+                {
+                    Id = "99333d69-806f-420a-932f-2cc23501f018",
+                    AeTitle = Helper.GetWorkflowByName("Workflow_Revision_for_export_folder").WorkflowRevision.Workflow.InformaticsGateway.AeTitle,
+                    WorkflowId = Helper.GetWorkflowByName("Workflow_Revision_for_export_folder").WorkflowRevision.WorkflowId,
+                    PayloadId = "fd1e99c1-341a-4400-aa28-3fa89d874968",
+                    StartTime = DateTime.Now,
+                    Status = Status.Created,
+                    BucketId = TestExecutionConfig.MinioConfig.Bucket,
+                    InputMetaData = new Dictionary<string, string>()
+                    {
+                        { "", "" }
+                    },
+                    Tasks = new List<TaskExecution>
+                    {
+                        new TaskExecution()
+                        {
+                            ExecutionId = "f8b6118e-bf4f-4a12-8a03-71f4a7469154",
+                            TaskId = Helper.GetWorkflowByName("Workflow_Revision_for_export_folder").WorkflowRevision.Workflow.Tasks[0].Id,
+                            TaskType = Helper.GetWorkflowByName("Workflow_Revision_for_export_folder").WorkflowRevision?.Workflow.Tasks[0].Type,
+                            Status = TaskExecutionStatus.Accepted,
+                            InputArtifacts = null,
+                            OutputArtifacts = null,
+                            OutputDirectory = "fd1e99c1-341a-4400-aa28-3fa89d874968/workflows/99333d69-806f-420a-932f-2cc23501f018/f8b6118e-bf4f-4a12-8a03-71f4a7469154",
+                        }
+                    }
+                }
+            },
+            new WorkflowInstanceTestData()
             {
                 Name = "Workflow_Instance_for_export_multi_dest_1",
                 WorkflowInstance = new WorkflowInstance()

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/TestData/WorkflowRevisionTestData.cs

@@ -2056,6 +2056,67 @@ public static class WorkflowRevisionsTestData
                 }
             },
             new WorkflowRevisionTestData()
+            {
+                Name = "Workflow_Revision_for_export_folder",
+                WorkflowRevision = new WorkflowRevision()
+                {
+                    Id = Guid.NewGuid().ToString(),
+                    WorkflowId = Guid.NewGuid().ToString(),
+                    Revision = 1,
+                    Workflow = new Workflow()
+                    {
+                        Name = "Artifact 1",
+                        Description = "Artifact 1",
+                        Version = "1",
+                        Tasks = new TaskObject[]
+                        {
+                            new TaskObject
+                            {
+                                Id = "artifact_task_1",
+                                Type = "Artifact_task",
+                                Description = "Artifact Workflow 1 Task 1",
+                                Artifacts = new ArtifactMap()
+                                {
+                                    Input = new Artifact[]
+                                    {
+                                        new Artifact { Name = "Dicom", Value = "{{ context.input.dicom }}" },
+                                    },
+                                },
+                                TaskDestinations = new TaskDestination[]
+                                {
+                                    new TaskDestination{ Name = "export_task_1" }
+                                }
+                            },
+                            new TaskObject
+                            {
+                                Id = "export_task_1",
+                                Type = "Export",
+                                Description = "Export Workflow 1 Task 2",
+                                ExportDestinations = new ExportDestination[]
+                                {
+                                    new ExportDestination { Name = "PROD_PACS" }
+                                },
+                                Artifacts = new ArtifactMap()
+                                {
+                                    Input = new Artifact[]
+                                    {
+                                        new Artifact { Name = "output", Value = "{{ context.executions.artifact_task_1.output_dir }}" },
+                                    },
+                                },
+                            },
+                        },
+                        InformaticsGateway = new InformaticsGateway()
+                        {
+                            AeTitle = "Artifact_AE",
+                            ExportDestinations = new string[]
+                            {
+                                "PROD_PACS"
+                            }
+                        }
+                    }
+                }
+            },
+            new WorkflowRevisionTestData()
             {
                 Name = "Workflow_Revision_for_export_multi_dest_1",
                 WorkflowRevision = new WorkflowRevision()