Feature k8s logging (#448)

* Update chart version for the changes to come.
* Create a specific congfigmap for logging.
* Update the templates to use the newly created logging configmap

deployment/entity-service/Chart.yaml

@@ -1,6 +1,6 @@
 name: entity-service
 appVersion: 1.11.2
-version: 1.12.0
+version: 1.13.0
 description: Privacy preserving record linkage service
 sources:
   - https://github.com/data61/anonlink-entity-service

deployment/entity-service/templates/api-deployment.yaml

@@ -94,6 +94,20 @@ spec:
             initialDelaySeconds: 60
             periodSeconds: 60
             timeoutSeconds: 5
+          {{- if .Values.loggingCfg }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /var/config
+          {{- end }}
+      {{- if .Values.loggingCfg }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ template "es.fullname" . }}-logging
+            items:
+              - key: "loggingCfg"
+                path: "logging_file.yaml"
+      {{- end }}
       {{- if .Values.api.pullSecret }}
       imagePullSecrets:
         - name: {{ .Values.api.pullSecret }}

deployment/entity-service/templates/configmap.yaml

@@ -66,7 +66,7 @@ data:
   MIN_ENCODING_SIZE: "8"
   MAX_ENCODING_SIZE: "1024"
 
+  {{- if .Values.loggingCfg }}
   # Not from the setting.py file.
-  {{ if .Values.workers.LOG_CFG }}
-  LOG_CFG: {{ .Values.workers.LOG_CFG | quote }}
-  {{ end }}
+  LOG_CFG: "/var/config/logging_file.yaml"
+  {{- end }}

deployment/entity-service/templates/highmemory-worker-deployment.yaml

@@ -59,7 +59,6 @@ spec:
                 secretKeyRef:
                   name: {{ template "es.fullname" . }}
                   key: minioSecretKey
-
           command:
             - "celery"
             - "-A"
@@ -70,10 +69,24 @@ spec:
             - "fair"
             - "-Q"
             - "celery,compute,highmemory"
+          {{- if .Values.loggingCfg }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /var/config
+          {{- end }}
           args:
           {{- range $key, $value := .Values.workers.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}
+      {{- if .Values.loggingCfg }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ template "es.fullname" . }}-logging
+            items:
+              - key: loggingCfg
+                path: "logging_file.yaml"
+      {{- end }}
       {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}

deployment/entity-service/templates/logging-configmap.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.loggingCfg }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "es.fullname" . }}-logging
+  labels:
+    {{- include "es.release_labels" . | indent 4 }}
+data:
+  loggingCfg: |-
+{{ .Values.loggingCfg | indent 4 }}
+{{- end }}

deployment/entity-service/templates/worker-deployment.yaml

@@ -59,7 +59,6 @@ spec:
                 secretKeyRef:
                   name: {{ template "es.fullname" . }}
                   key: minioSecretKey
-
           command:
             - "celery"
             - "-A"
@@ -70,10 +69,24 @@ spec:
             - "fair"
             - "-Q"
             - "celery,compute"
+          {{- if .Values.loggingCfg }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /var/config
+          {{- end }}
           args:
           {{- range $key, $value := .Values.workers.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}
+      {{- if .Values.loggingCfg }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ template "es.fullname" . }}-logging
+            items:
+              - key: loggingCfg
+                path: "logging_file.yaml"
+      {{- end }}
       {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}

deployment/entity-service/values.yaml

@@ -172,11 +172,6 @@ workers:
   ## Default is 30 days:
   CACHE_EXPIRY_SECONDS: "2592000"
 
-  ## Optional path to a logging configuration file
-  ## Eventually this config can be stored in a configmap
-  ## See https://github.com/data61/anonlink-entity-service/issues/323
-  #LOG_CFG: "entityservice/verbose_logging.yaml"
-
   ## Specific configuration for celery
   ## Note that these configurations are the same for a "normal" worker, and a "highmemeroy" one,
   ## except for the requested resources and replicaCount which can differ.
@@ -329,3 +324,29 @@ provision:
   minio: true
   postgresql: true
   redis: true
+
+## Custom logging file used to override the default settings. Will be used by the workers and the flask container.
+## Example of logging configuration:
+#loggingCfg: |-
+#  version: 1
+#  disable_existing_loggers: False
+#  formatters:
+#    simple:
+#      format: "%(message)s"
+#    file:
+#      format: "%(asctime)-15s %(name)-12s %(levelname)-8s: %(message)s"
+#  filters:
+#    stderr_filter:
+#      (): entityservice.logger_setup.StdErrFilter
+#    stdout_filter:
+#      (): entityservice.logger_setup.StdOutFilter
+#  handlers:
+#    stdout:
+#      class: logging.StreamHandler
+#      level: DEBUG
+#      formatter: simple
+#      filters: [stdout_filter]
+#      stream: ext://sys.stdout
+#  root:
+#    level: INFO
+#    handlers: [stdout]

docs/changelog.rst

@@ -7,6 +7,9 @@ Changelog
 Next Version
 ------------
 
+Logging configurable in the deployed entity service by using the key ``loggingCfg``.
+
+
 Several old settings have been removed from the default values.yaml and docker files:
 - ``SMALL_COMPARISON_CHUNK_SIZE``
 - ``LARGE_COMPARISON_CHUNK_SIZE``