Use NLog (#198)

Signed-off-by: Victor Chang <vicchang@nvidia.com>
Co-authored-by: Neil South <neil.south@answerdigital.com>

.github/containerscan/allowedlist.yaml

@@ -15,3 +15,4 @@
 general:
   vulnerabilities:
     - CVE-2018-8292
+    - CVE-1999-1278

.licenserc.yaml

@@ -33,6 +33,8 @@ header:
     - 'src/Database/Migrations/**'
     - 'demos/**/.env/**'
     - 'demos/**/*.txt'
+    - 'doc/dependency_decisions.yml'
+    - 'docker-compose/configs/kibana.ndjson'
     - 'docs/templates/**'
     - 'tests/Integration.Test/*.dev'
     - 'tests/Integration.Test/data/**'

Dockerfile

@@ -28,7 +28,10 @@ RUN dotnet publish -c Release -o out --nologo src/InformaticsGateway/Monai.Deplo
 # Build runtime image
 FROM mcr.microsoft.com/dotnet/aspnet:6.0-jammy
 
+# Enable elastic client compatibility mode
+ENV ELASTIC_CLIENT_APIVERSIONING=true
 ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get clean \
  && apt-get update \
  && apt-get install -y --no-install-recommends \

doc/dependency_decisions.yml

@@ -46,7 +46,7 @@
   - :who: mocsharp
     :why: MIT (https://github.com/Cysharp/ConsoleAppFramework/raw/master/LICENSE)
     :versions:
-    - 4.2.3
+    - 4.2.4
     :when: 2022-08-16 23:05:31.110052610 Z
 - - :approve
   - Crayon
@@ -808,14 +808,14 @@
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
     :versions:
-    - 0.1.6
+    - 0.1.8
     :when: 2022-08-16 23:06:21.051573547 Z
 - - :approve
   - Monai.Deploy.Messaging.RabbitMQ
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
     :versions:
-    - 0.1.6
+    - 0.1.8
     :when: 2022-08-16 23:06:21.511789690 Z
 - - :approve
   - Monai.Deploy.Storage
@@ -1694,7 +1694,6 @@
     :versions:
     - 4.7.2
     :when: 2022-08-16 23:07:19.845361666 Z
-
 - - :approve
   - System.Text.Json
   - :who: mocsharp
@@ -2370,3 +2369,31 @@
     :versions:
     - 4.0.1
     :when: 2022-08-16 23:10:21.184627612 Z
+- - :approve
+  - NLog
+  - :who: mocsharp
+    :why: BSD 3-Clause License (https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
+    :versions:
+    - 5.0.4
+    :when: 2022-10-12 03:14:06.538744982 Z
+- - :approve
+  - NLog.Extensions.Logging
+  - :who: mocsharp
+    :why: BSD 2-Clause Simplified License (https://github.com/NLog/NLog.Extensions.Logging/raw/master/LICENSE)
+    :versions:
+    - 5.0.4
+    :when: 2022-10-12 03:14:06.964203977 Z
+- - :approve
+  - NLog.Web.AspNetCore
+  - :who: mocsharp
+    :why: BSD 3-Clause License (https://github.com/NLog/NLog.Web/raw/master/LICENSE)
+    :versions:
+    - 5.1.4
+    :when: 2022-10-12 03:14:07.396706995 Z
+- - :approve
+  - fo-dicom.NLog
+  - :who: mocsharp
+    :why: Microsoft Public License (https://github.com/fo-dicom/fo-dicom/raw/development/License.txt)
+    :versions:
+    - 5.0.3
+    :when: 2022-10-12 03:14:08.789273776 Z

docker-compose/.env

@@ -0,0 +1,28 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ELASTIC_VERSION=8.4.2
+
+
+# Elastic Search
+# User 'elastic' (built-in)
+ELASTIC_PASSWORD='monai'
+
+# Logstash
+# User 'logstash_internal' (custom)
+LOGSTASH_INTERNAL_PASSWORD='monai'
+
+# Kibana
+# User 'kibana_system' (built-in)
+KIBANA_SYSTEM_PASSWORD='monai'

docker-compose/README.md

@@ -0,0 +1,42 @@
+<!-- 
+Copyright 2022 MONAI Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. 
+-->
+
+# MONAI Deploy Informatics Gateway
+
+The [docker-compose.yml](./docker-compose.yml) file includes the following services to run the Informatics Gateway.
+
+* MinIO
+* RabbitMQ
+* ElasticSearch (optional)
+* LogStash (optional)
+* Kibana (optional)
+
+
+## Running docker compose
+
+To start all Informatics Gateway dependencies, run `docker compose up`.
+
+Before running Informatics Gateway, ensure the following environment variables are exported:
+
+```bash
+export DOTNET_ENVIRONMENT=Development # if using appsettings.Development.json
+export LOGSTASH_URL=tcp://localhost:50000 # this tells Informatics Gateway to export logs to LogStash at tcp://localhost:5000
+```
+IMPORTANT: for Linux users, before running `docker compose up`, please run `init.sh` to create directories with the correct permissions first. Otherwise, ElasticSearch will not be able to start.
+
+### Kibana
+
+A default search is imported to Kibana at startup. To load the saved search, go to Analytics > Discover from the 🍔 menu. From the top right click *Open* and select *MONAI-Default*.

docker-compose/configs/elasticsearch.yml

@@ -0,0 +1,27 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+## Default Elasticsearch configuration from Elasticsearch base image.
+## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml
+#
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+## X-Pack settings
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html
+#
+xpack.security.enabled: false
+xpack.license.self_generated.type: basic

docker-compose/configs/kibana.ndjson

@@ -0,0 +1,3 @@
+{"attributes":{"fieldAttrs":"{\"@version\":{\"count\":1},\"EventId\":{\"count\":3},\"Level\":{\"count\":4},\"LoggerName\":{\"count\":3},\"MachineName\":{\"count\":1},\"Message\":{\"count\":2},\"ServiceName\":{\"count\":1},\"ServiceVersion\":{\"count\":1}}","fields":"[]","name":"monai-deploy","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"monai-deploy-*","typeMeta":"{}"},"coreMigrationVersion":"8.4.2","id":"6c4ef8fa-f382-4dad-8c82-f180f09fc88d","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-10-11T21:54:09.450Z","version":"Wzc2MiwxXQ=="}
+{"attributes":{"columns":["EventId","Level","LoggerName","Message"],"description":"","grid":{"columns":{"EventId":{"width":124},"Level":{"width":67.66666666666663},"LoggerName":{"width":668.1666666666667}}},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"Level\",\"params\":[\"TRACE\"],\"alias\":null,\"negate\":true,\"disabled\":false,\"value\":[\"TRACE\"],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"Level\":\"TRACE\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"MONAI-Default"},"coreMigrationVersion":"8.4.2","id":"48ce7db0-49af-11ed-bde2-430e63791a09","migrationVersion":{"search":"8.0.0"},"references":[{"id":"6c4ef8fa-f382-4dad-8c82-f180f09fc88d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"6c4ef8fa-f382-4dad-8c82-f180f09fc88d","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2022-10-11T21:54:30.795Z","version":"Wzc3NSwxXQ=="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]}

docker-compose/configs/kibana.yml

@@ -0,0 +1,27 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+## Default Kibana configuration from Kibana base image.
+## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts
+#
+server.name: kibana
+server.host: 0.0.0.0
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+monitoring.ui.container.elasticsearch.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: kibana_system
+elasticsearch.password: monai

docker-compose/configs/logstash.yml

@@ -0,0 +1,18 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+## Default Logstash configuration from Logstash base image.
+## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml
+#
+http.host: "0.0.0.0"

docker-compose/configs/pipeline.yml

@@ -0,0 +1,16 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- pipeline.id: monei-deploy
+  path.config: "/usr/share/logstash/pipeline/monai-deploy.conf"
+  queue.type: persisted

docker-compose/configs/pipeline/monai-deploy.conf

@@ -0,0 +1,47 @@
+# Copyright 2022 MONAI Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+input {
+        tcp {
+            port => 50000
+            type => nlog
+        }
+}
+
+filter {
+    if [type] == "nlog" {
+            mutate {
+                remove_field => [ "host" ]
+            }
+            json {
+                source => "message"
+                remove_field => [ "message" ]
+            }
+            date{
+                match => [ "Timestamp", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ"  ]
+            }
+
+            mutate {
+                remove_field => [ "time"  ]
+            }		  
+    }
+}
+
+output {
+  elasticsearch {
+        hosts => "elasticsearch:9200"
+        index => "monai-deploy-%{+YYYY.MM.dd}"
+  }
+}