#391 nds addlogging

deploy/helm/MTM.yaml

@@ -27,9 +27,12 @@ enviromentVariables:
   WorkflowManager__storage__settings__accessKey: "rootminio"
   WorkflowManager__storage__settings__accessToken: "rootminio"
   WorkflowManager__storage__settings__executableLocation: "mc"
-  WorkflowManager__messaging__publisherSettings__endpoint: "rabbitmq.monai"
-  WorkflowManager__messaging__subscriberSettings__endpoint: "rabbitmq.monai"
-  WorkloadManagerDatabase__ConnectionString: "mongodb://root:rootpassword@mongo-monai.monai:27017"
+  WorkflowManager__messaging__publisherSettings__endpoint: "rabbit-monai.argo"
+  WorkflowManager__messaging__publisherSettings__port: "5672"  
+  WorkflowManager__messaging__subscriberSettings__endpoint: "rabbit-monai.argo"
+  WorkflowManager__messaging__subscriberSettings__port: "5672"
+  Serilog__WriteTo__0__Args__indexFormat: "index-monai{0:yyyy.MM}"
+  Serilog__WriteTo__0__Args__nodeUris: "http://192.168.0.22:9200"
 
 enviromentVariablesFromSecrets:
   secrets:

guidelines/mwm-developer-setup.md

@@ -74,7 +74,7 @@ Open the post/workflows tab and click `try it out`, paste in the following to th
 	"version": "1.0.0",
 	"description": "Attempt at making a workflow",
 	"informatics_gateway": {
-		"ae_title": "MonaiSCU",
+		"ae_title": "MONAISCU",
 		"data_origins": [
 			"MY_SCANNER"
 		],
@@ -91,25 +91,19 @@ Open the post/workflows tab and click `try it out`, paste in the following to th
 				"namespace":"argo",
 				"workflow_template_name": "simple-workflow",
 				"server_url": "https://localhost:2746",
-				"allow_insecure": true,
-				"messaging_endpoint": "rabbit-monai",
-                "messaging_username" : "admin",
-                "messaging_password" : "admin",
-                "messaging_topic" : "md.tasks.callback",
-                "messaging_exchange" : "monaideploy",
-				"messaging_vhost" : "monaideploy",
+				"allow_insecure": true
 			},
 			"artifacts": {
 				"input": [
 					{
 						"name": "input_diacom",
-						"value": "{{ context.input.dicom }}/input_dicom"
+						"value": "{{ context.input.dicom }}"
 					}
 				],
 				"output": [
 					{
 						"name": "report",
-						"value": "{{ context.output }}/report",
+						"value": "{{ context.output }}",
 						"Mandatory": false
 					}
 				]

src/TaskManager/Database/Logging/Log.1000.Database.cs

@@ -23,10 +23,10 @@ public static partial class Log
         [LoggerMessage(EventId = 1000, Level = LogLevel.Error, Message = "Database call failed in {methodName}.")]
         public static partial void DatabaseException(this ILogger logger, string methodName, Exception ex);
 
-        [LoggerMessage(EventId = 1001, Level = LogLevel.Information, Message = "Task dispatch event saved {taskExecutionId}.")]
-        public static partial void TaskDispatchEventSaved(this ILogger logger, string taskExecutionId);
+        [LoggerMessage(EventId = 1001, Level = LogLevel.Information, Message = "Task dispatch event saved {executionId}.")]
+        public static partial void TaskDispatchEventSaved(this ILogger logger, string executionId);
 
-        [LoggerMessage(EventId = 1002, Level = LogLevel.Information, Message = "Task dispatch event deleted {taskExecutionId}.")]
-        public static partial void TaskDispatchEventDeleted(this ILogger logger, string taskExecutionId);
+        [LoggerMessage(EventId = 1002, Level = LogLevel.Information, Message = "Task dispatch event deleted {executionId}.")]
+        public static partial void TaskDispatchEventDeleted(this ILogger logger, string executionId);
     }
 }

src/TaskManager/Database/packages.lock.json

@@ -13,9 +13,9 @@
       },
       "Microsoft.Extensions.Logging.Abstractions": {
         "type": "Direct",
-        "requested": "[6.0.1, )",
-        "resolved": "6.0.1",
-        "contentHash": "dzB2Cgg+JmrouhjkcQGzSFjjvpwlq353i8oBQO2GWNjCXSzhbtBRUf28HSauWe7eib3wYOdb3tItdjRwAdwCSg=="
+        "requested": "[6.0.2, )",
+        "resolved": "6.0.2",
+        "contentHash": "pwXCZKaA7m5wgmCj49dW+H1RPSY7U62SKLTQYCcavf/k3Nyt/WnBgAjG4jMGnwy9rElfAZ2KvxvM5CJzJWG0hg=="
       },
       "Microsoft.Extensions.Options": {
         "type": "Direct",

src/TaskManager/Plug-ins/Argo/ArgoPlugin.cs

@@ -129,7 +129,13 @@ private void ValidateEvent()
 
         public override async Task<ExecutionStatus> ExecuteTask(CancellationToken cancellationToken = default)
         {
-            using var loggerScope = _logger.BeginScope($"Workflow ID={Event.WorkflowInstanceId}, Task ID={Event.TaskId}, Execution ID={Event.ExecutionId}, Argo namespace={_namespace}");
+            using var loggingScope = _logger.BeginScope(new Dictionary<string, object>
+            {
+                ["correlationId"] = Event.CorrelationId,
+                ["workflowInstanceId"] = Event.WorkflowInstanceId,
+                ["taskId"] = Event.TaskId,
+                ["argoNamespace"] = _namespace
+            });
 
             Workflow workflow;
             try
@@ -237,7 +243,7 @@ private Dictionary<string, string> GetExecutuionStats(Workflow workflow)
             }
             var stats = new Dictionary<string, string>
             {
-                { "workflowId", Event.WorkflowInstanceId },
+                { "workflowInstanceId", Event.WorkflowInstanceId },
                 { "duration", duration.HasValue ? duration.Value.TotalMilliseconds.ToString() : string.Empty },
                 { "startedAt", workflow.Status?.StartedAt.ToString() ?? string.Empty  },
                 { "finishedAt", workflow.Status?.FinishedAt.ToString() ?? string.Empty  }

src/TaskManager/Plug-ins/Argo/packages.lock.json

@@ -28,9 +28,9 @@
       },
       "Microsoft.Extensions.ApiDescription.Client": {
         "type": "Direct",
-        "requested": "[6.0.8, )",
-        "resolved": "6.0.8",
-        "contentHash": "fN3zEuHjSql9w2Md29dYkmPbAijxr+LqxoOoJ9B8G2j+fHgAK/N0g89miIUwepwbk8izTi5m8cVyWTKQgkd+ww=="
+        "requested": "[6.0.9, )",
+        "resolved": "6.0.9",
+        "contentHash": "DjwJLPdOzBceBV+V91FfFGHGV2I0UKfOpCLtCXQhaSdkyDF/0Ce2j0GnNrqYesU/LJ9R6n6mIU8fgieD5absMg=="
       },
       "Monai.Deploy.Messaging": {
         "type": "Direct",
@@ -416,8 +416,8 @@
       },
       "System.IO.Abstractions": {
         "type": "Transitive",
-        "resolved": "17.1.1",
-        "contentHash": "LWOM12Bd0kp/gaH4g1o/O2/6JDcHF/fuctF1IqDZt0aAqU2BwGiMihi9Cdcm5jJz8La1wFWbRPuRui8WeX6m8w=="
+        "resolved": "17.2.3",
+        "contentHash": "VcozGeE4SxIo0cnXrDHhbrh/Gb8KQnZ3BvMelvh+iw0PrIKtuuA46U2Xm4e4pgnaWFgT4RdZfTpWl/WPRdw0WQ=="
       },
       "System.Memory": {
         "type": "Transitive",

src/TaskManager/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj

@@ -1,108 +1,114 @@
-<!--
-  ~ Copyright 2021-2022 MONAI Consortium
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~ http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
--->
-
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <FrameworkReference Include="Microsoft.AspNetCore.App" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Compile Remove="API\**" />
-    <EmbeddedResource Remove="API\**" />
-    <None Remove="API\**" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Compile Remove="Plug-ins\**" />
-    <EmbeddedResource Remove="Plug-ins\**" />
-    <None Remove="Plug-ins\**" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Compile Remove="Database\**" />
-    <EmbeddedResource Remove="Database\**" />
-    <None Remove="Database\**" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
-    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
-    <PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.6-rc0001" />
-    <PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.5" />
-    <PackageReference Include="Monai.Deploy.Messaging.RabbitMQ" Version="0.1.6-rc0001" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\Shared\Configuration\Monai.Deploy.WorkflowManager.Configuration.csproj" />
-    <ProjectReference Include="..\API\Monai.Deploy.WorkflowManager.TaskManager.API.csproj" />
-    <ProjectReference Include="..\Database\Monai.Deploy.WorkflowManager.TaskManager.Database.csproj" />
-    <ProjectReference Include="..\Plug-ins\Argo\Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj" />
-    <ProjectReference Include="..\Plug-ins\Docker\Monai.Deploy.WorkflowManager.TaskManager.Docker.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <None Update="appsettings.Development.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="appsettings.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="appsettings.Local.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-  <PropertyGroup>
-    <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
-    <DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
-    <CodeAnalysisRuleSet>..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-
-  <Target Name="CopyPlugins" AfterTargets="publish">
-    <ItemGroup>
-      <PluginDlls Include="$(PublishDir)Monai.Deploy.Messaging.RabbitMQ.dll;$(PublishDir)Monai.Deploy.Storage.MinIO.dll" />
-    </ItemGroup>
-    <Copy SourceFiles="@(PluginDlls)" DestinationFolder="$(PublishDir)\plug-ins\" SkipUnchangedFiles="true" />
-    <Message Text="Files copied successfully to $(PublishDir)\plug-ins\." Importance="high" />
-  </Target>
-
-  <Target Name="CopyPluginsBuild" AfterTargets="build">
-    <ItemGroup>
-      <PluginDlls Include="$(OutDir)Monai.Deploy.Messaging.RabbitMQ.dll;$(OutDir)Monai.Deploy.Storage.MinIO.dll" />
-    </ItemGroup>
-    <Copy SourceFiles="@(PluginDlls)" DestinationFolder="$(OutDir)\plug-ins\" SkipUnchangedFiles="true" />
-    <Message Text="Files copied successfully to $(OutDir)\plug-ins\." Importance="high" />
-  </Target>
-
-</Project>
+<!--
+  ~ Copyright 2021-2022 MONAI Consortium
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Remove="API\**" />
+    <EmbeddedResource Remove="API\**" />
+    <None Remove="API\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Remove="Plug-ins\**" />
+    <EmbeddedResource Remove="Plug-ins\**" />
+    <None Remove="Plug-ins\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Remove="Database\**" />
+    <EmbeddedResource Remove="Database\**" />
+    <None Remove="Database\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Elastic.CommonSchema" Version="1.5.3" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
+    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
+    <PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.6-rc0001" />
+    <PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.5" />
+    <PackageReference Include="Monai.Deploy.Messaging.RabbitMQ" Version="0.1.6-rc0001" />
+    <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />
+    <PackageReference Include="Elastic.CommonSchema.Serilog" Version="1.5.3" />
+    <PackageReference Include="Serilog.Sinks.Http" Version="8.0.0" />
+    <PackageReference Include="Serilog.Enrichers.Environment" Version="2.2.0" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Shared\Configuration\Monai.Deploy.WorkflowManager.Configuration.csproj" />
+    <ProjectReference Include="..\API\Monai.Deploy.WorkflowManager.TaskManager.API.csproj" />
+    <ProjectReference Include="..\Database\Monai.Deploy.WorkflowManager.TaskManager.Database.csproj" />
+    <ProjectReference Include="..\Plug-ins\Argo\Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj" />
+    <ProjectReference Include="..\Plug-ins\Docker\Monai.Deploy.WorkflowManager.TaskManager.Docker.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.Development.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="appsettings.Local.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+  <PropertyGroup>
+    <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
+    <DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
+    <CodeAnalysisRuleSet>..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+
+  <Target Name="CopyPlugins" AfterTargets="publish">
+    <ItemGroup>
+      <PluginDlls Include="$(PublishDir)Monai.Deploy.Messaging.RabbitMQ.dll;$(PublishDir)Monai.Deploy.Storage.MinIO.dll" />
+    </ItemGroup>
+    <Copy SourceFiles="@(PluginDlls)" DestinationFolder="$(PublishDir)\plug-ins\" SkipUnchangedFiles="true" />
+    <Message Text="Files copied successfully to $(PublishDir)\plug-ins\." Importance="high" />
+  </Target>
+
+  <Target Name="CopyPluginsBuild" AfterTargets="build">
+    <ItemGroup>
+      <PluginDlls Include="$(OutDir)Monai.Deploy.Messaging.RabbitMQ.dll;$(OutDir)Monai.Deploy.Storage.MinIO.dll" />
+    </ItemGroup>
+    <Copy SourceFiles="@(PluginDlls)" DestinationFolder="$(OutDir)\plug-ins\" SkipUnchangedFiles="true" />
+    <Message Text="Files copied successfully to $(OutDir)\plug-ins\." Importance="high" />
+  </Target>
+
+</Project>

src/TaskManager/TaskManager/Program.cs

@@ -15,6 +15,8 @@
  */
 
 using System.IO.Abstractions;
+using System.Reflection;
+using Elastic.CommonSchema.Serilog;
 using Microsoft.AspNetCore.StaticFiles;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -29,6 +31,9 @@
 using Monai.Deploy.WorkflowManager.TaskManager.Database.Options;
 using Monai.Deploy.WorkflowManager.TaskManager.Extensions;
 using MongoDB.Driver;
+using Serilog;
+using Serilog.Events;
+using Serilog.Exceptions;
 
 namespace Monai.Deploy.WorkflowManager.TaskManager
 {
@@ -64,6 +69,21 @@ public static IHostBuilder CreateHostBuilder(string[] args) =>
                     configureLogging.AddConfiguration(builderContext.Configuration.GetSection("Logging"));
                     configureLogging.AddFile(o => o.RootPath = AppContext.BaseDirectory);
                 })
+                .UseSerilog((context, services, configuration) => configuration
+                    .ReadFrom.Configuration(context.Configuration)
+                    .ReadFrom.Services(services)
+                    .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
+                    .MinimumLevel.Debug()
+                    .Enrich.FromLogContext()
+                    .Enrich.WithExceptionDetails()
+                    .Enrich.WithProperty("dllversion", Assembly.GetEntryAssembly()?.GetName().Version)
+                    .Enrich.WithProperty("dllName", Assembly.GetEntryAssembly()?.GetName().Name)
+                    .WriteTo.File(
+                        path: "logs/MTM-.log",
+                        rollingInterval: RollingInterval.Day,
+                        formatter: new EcsTextFormatter())
+                .WriteTo.Console())
+
                 .ConfigureServices((hostContext, services) =>
                 {
                     ConfigureServices(hostContext, services);