add support for custom index name prefix for sidecar logging (#2045)

* add support for custom index name prefix for sidecar logging * add sidecar logging custom index prefix * fix pre-commit * fix kibana index job conditionals * fix typo error * index config rework fluentd and sidecar * fix pre-commit failures * code cleanp for helm templates * add index prefix override for fluentd service * fluentd test case cleanup * update test cases for fluentd * move conditionals to templates * added relavent test cases index prefix overrides * fix houston configmap and update test cases * fix pre-commit * fix fluentd config issue * move code duplicates to common helper template * fix pre-commit failures * Update tests/chart_tests/test_houston_configmap.py Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * Update tests/chart_tests/test_houston_configmap.py Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * Update tests/chart_tests/test_houston_configmap.py Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * Update tests/chart_tests/test_houston_configmap.py Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * fix failing tests and updates * Update charts/astronomer/templates/_helpers.yaml Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * add extra test cases for fluentd flag in houston --------- Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com>
astronomer · Nov 17, 2023 · 67c8bdf · 67c8bdf
1 parent dfbfb1b
commit 67c8bdf
Show file tree

Hide file tree

Showing 16 changed files with 523 additions and 294 deletions.
diff --git a/charts/astronomer/templates/_helpers.yaml b/charts/astronomer/templates/_helpers.yaml
@@ -378,3 +378,11 @@ imagePullSecrets:
   - name: {{ .Values.global.privateRegistry.secretName }}
 {{- end -}}
 {{- end -}}
+
+{{ define "fluentd.IndexPattern" -}}
+{{- if .Values.global.logging.indexNamePrefix -}}
+{{ .Values.global.logging.indexNamePrefix }}
+{{- else -}}
+fluentd
+{{- end -}}
+{{- end }}
diff --git a/charts/astronomer/templates/houston/houston-configmap.yaml b/charts/astronomer/templates/houston/houston-configmap.yaml
@@ -57,6 +57,7 @@ data:
 
     # Airflow deployment configuration
     deployments:
+      fluentdIndexPrefix: {{ include "fluentd.IndexPattern" .}}
       enableHoustonInternalAuthorization: {{ .Values.houston.enableHoustonInternalAuthorization }}
       namespaceFreeFormEntry: {{ .Values.global.namespaceFreeFormEntry }}
       # Airflow chart settings
@@ -120,6 +121,9 @@ data:
         {{- if .Values.global.loggingSidecar.indexPattern }}
         indexPattern: {{ .Values.global.loggingSidecar.indexPattern | squote }}
         {{- end }}
+        {{- if .Values.global.logging.indexNamePrefix }}
+        indexNamePrefix: {{ .Values.global.logging.indexNamePrefix }}
+        {{- end }}
       {{- end }}
 
       # These values get passed directly into the airflow helm deployments

diff --git a/charts/elasticsearch/templates/nginx/nginx-es-configmap.yaml b/charts/elasticsearch/templates/nginx/nginx-es-configmap.yaml
@@ -39,29 +39,17 @@ data:
         # The following "location" rules limit airflow interactions to only their indices. Any further
         # additions should follow this pattern.
         location ~* /_count$ {
-          {{- if .Values.global.loggingSidecar.enabled }}
-          rewrite /_count(.*) /vector.$remote_user.*/_count$1 break;
-          {{- else }}
-          rewrite /_count(.*) /fluentd.$remote_user.*/_count$1 break;
-          {{- end }}
+          rewrite /_count(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/_count$1 break;
           proxy_pass http://elasticsearch;
         }
 
         location ~* /_bulk$ {
-          {{- if .Values.global.loggingSidecar.enabled }}
-          rewrite /_bulk(.*) /vector.$remote_user.*/_bulk$1 break;
-          {{- else }}
-          rewrite /_bulk(.*) /fluentd.$remote_user.*/_bulk$1 break;
-          {{- end }}
+          rewrite /_bulk(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/_bulk$1 break;
           proxy_pass http://elasticsearch;
         }
 
         location ~* /_search$ {
-          {{- if .Values.global.loggingSidecar.enabled }}
-          rewrite /_search(.*) /vector.$remote_user.*/_search$1 break;
-          {{- else }}
-          rewrite /_search(.*) /fluentd.$remote_user.*/_search$1 break;
-          {{- end }}
+          rewrite /_search(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/_search$1 break;
           proxy_pass http://elasticsearch;
         }
 

diff --git a/charts/external-es-proxy/templates/external-es-proxy-configmap.yaml b/charts/external-es-proxy/templates/external-es-proxy-configmap.yaml
@@ -44,11 +44,7 @@ data:
         location = /_search {
           # This combined with disabling explicit index searching downstream
           # prevents any deployment from being able to query any other indexes.
-          {{- if .Values.global.loggingSidecar.enabled }}
-          rewrite ^/(.*) /vector.$remote_user.*/$1 break;
-          {{- else }}
-          rewrite ^/(.*) /fluentd.$remote_user.*/$1 break;
-          {{- end }}
+          rewrite ^/(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/$1 break;
           {{- if  or .Values.global.customLogging.awsSecretName  .Values.global.customLogging.awsServiceAccountAnnotation .Values.global.customLogging.awsIAMRole }}
           proxy_pass http://localhost:{{ .Values.service.awsproxy }};
           {{- else }}

diff --git a/charts/fluentd/templates/_helpers.tpl b/charts/fluentd/templates/_helpers.tpl
@@ -85,3 +85,11 @@ imagePullSecrets:
   - name: {{ .Values.global.privateRegistry.secretName }}
 {{- end -}}
 {{- end -}}
+
+{{- define "fluentd.indexNamePrefix" -}}
+{{- if .Values.global.logging.indexNamePrefix -}}
+{{ .Values.global.logging.indexNamePrefix }}
+{{- else -}}
+fluentd
+{{- end -}}
+{{- end -}}
diff --git a/charts/fluentd/templates/fluentd-configmap.yaml b/charts/fluentd/templates/fluentd-configmap.yaml
@@ -204,7 +204,7 @@ data:
         suppress_type_name true
         host "#{ENV['OUTPUT_HOST']}"
         port "#{ENV['OUTPUT_PORT']}"
-        index_name fluentd.${record["release"]}.${Time.at(time).getutc.strftime(@logstash_dateformat)}
+        index_name {{ include "fluentd.indexNamePrefix" .}}.${record["release"]}.${Time.at(time).getutc.strftime(@logstash_dateformat)}
         templates { "fluentdindextemplate": "/host/index_template.json"}
         template_overwrite true
         <buffer>

diff --git a/charts/fluentd/templates/fluentd-index-template-configmap.yaml b/charts/fluentd/templates/fluentd-index-template-configmap.yaml
@@ -15,7 +15,7 @@ data:
   index_template.json: |-
     {
       "index_patterns": [
-          "fluentd.*"
+          "{{ include "fluentd.indexNamePrefix" .}}.*"
       ],
       "mappings": {
         "properties": {

diff --git a/charts/kibana/templates/_helpers.tpl b/charts/kibana/templates/_helpers.tpl
@@ -63,7 +63,3 @@ imagePullSecrets:
   - name: {{ .Values.global.privateRegistry.secretName }}
 {{- end -}}
 {{- end -}}
-
-{{ define "kibana.IndexPattern" -}}
-{{- if .Values.global.loggingSidecar.enabled }}vector{{- else }}fluentd{{- end -}}
-{{- end }}
diff --git a/charts/kibana/templates/kibana-default-index-cronjob.yaml b/charts/kibana/templates/kibana-default-index-cronjob.yaml
@@ -38,12 +38,12 @@ spec:
           - "/bin/sh"
           - -ec
           - |
-            http_status=$(curl --retry 10 --retry-delay 30 --retry-all-errors -s -o /dev/null -w "%{http_code}" -XGET http://{{ .Release.Name }}-kibana:5601/api/data_views/data_view/{{ template "kibana.IndexPattern" . }}.*)
+            http_status=$(curl --retry 10 --retry-delay 30 --retry-all-errors -s -o /dev/null -w "%{http_code}" -XGET http://{{ .Release.Name }}-kibana:5601/api/data_views/data_view/{{ template "logging.indexNamePrefix" . }}.*)
             if [[ "$http_status" -eq 200 ]]; then
               echo "Kibana Index Pattern Already Exists. Skipping Creation"
             else
               echo "Creating Kibana Index Pattern"
-              curl --retry 10 --retry-delay 30 --retry-all-errors -XPOST -H 'Content-Type: application/json' -H 'kbn-xsrf: astronomer' --data '{"attributes":{"title":"{{ template "kibana.IndexPattern" . }}.*","timeFieldName":"@timestamp"}}' http://{{ .Release.Name }}-kibana:5601/api/saved_objects/index-pattern/{{ template "kibana.IndexPattern" . }}.*?overwrite=false
+              curl --retry 10 --retry-delay 30 --retry-all-errors -XPOST -H 'Content-Type: application/json' -H 'kbn-xsrf: astronomer' --data '{"attributes":{"title":"{{ template "logging.indexNamePrefix" . }}.*","timeFieldName":"@timestamp"}}' http://{{ .Release.Name }}-kibana:5601/api/saved_objects/index-pattern/{{ template "logging.indexNamePrefix" . }}.*?overwrite=false
             fi
         restartPolicy: Never
 {{ end }}
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -0,0 +1,11 @@
+{{- define "logging.indexNamePrefix" -}}
+{{- if .Values.global.logging.indexNamePrefix -}}
+{{- .Values.global.logging.indexNamePrefix -}}
+{{- else -}}
+{{- if .Values.global.loggingSidecar.enabled  -}}
+vector
+{{- else -}}
+fluentd
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/tests/chart_tests/test_elasticsearch.py b/tests/chart_tests/test_elasticsearch.py
@@ -263,7 +263,7 @@ def test_nginx_es_client_network_selector_with_logging_sidecar_enabled(
             },
         ] == doc["spec"]["ingress"][0]["from"]
 
-    def test_nginx_nginx_config_pattern_defaults(self, kube_version):
+    def test_elastic_nginx_config_pattern_defaults(self, kube_version):
         """Test External Elasticsearch Service Index Pattern Search
         defaults."""
         docs = render_chart(
@@ -291,7 +291,36 @@ def test_nginx_nginx_config_pattern_defaults(self, kube_version):
             ]
         )
 
-    def test_nginx_nginx_config_pattern_with_sidecar_logging_enabled(
+    def test_elastic_nginx_config_pattern_defaults_and_index_prefix_overrides(
+        self, kube_version
+    ):
+        """Test External Elasticsearch Service Index Pattern Search with index prefix overrides."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={"global": {"logging": {"indexNamePrefix": "astronomer"}}},
+            show_only=[
+                "charts/elasticsearch/templates/nginx/nginx-es-configmap.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        assert doc["kind"] == "ConfigMap"
+
+        nginx_config = " ".join(doc["data"]["nginx.conf"].split())
+        assert all(
+            x in nginx_config
+            for x in [
+                "location ~* /_bulk$ { rewrite /_bulk(.*) /astronomer.$remote_user.*/_bulk$1 break;",
+                "location ~* /_count$ { rewrite /_count(.*) /astronomer.$remote_user.*/_count$1 break;",
+                "location ~* /_search$ { rewrite /_search(.*) /astronomer.$remote_user.*/_search$1 break;",
+                "location = /_cluster/health { proxy_pass http://elasticsearch; }",
+                "location = /_cluster/state/version { proxy_pass http://elasticsearch; }",
+                "location ~ ^/ { deny all; } } }",
+            ]
+        )
+
+    def test_elasticsearch_nginx_config_pattern_with_sidecar_logging_enabled(
         self, kube_version
     ):
         """Test Nginx ES Service Index Pattern Search with sidecar logging."""
@@ -320,6 +349,40 @@ def test_nginx_nginx_config_pattern_with_sidecar_logging_enabled(
             ]
         )
 
+    def test_elasticsearch_nginx_config_pattern_with_sidecar_logging_enabled_and_index_prefix_overrides(
+        self, kube_version
+    ):
+        """Test Nginx ES Service Index Pattern Search with sidecar logging."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={
+                "global": {
+                    "loggingSidecar": {"enabled": True},
+                    "logging": {"indexNamePrefix": "astronomer"},
+                }
+            },
+            show_only=[
+                "charts/elasticsearch/templates/nginx/nginx-es-configmap.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        assert doc["kind"] == "ConfigMap"
+
+        nginx_config = " ".join(doc["data"]["nginx.conf"].split())
+        assert all(
+            x in nginx_config
+            for x in [
+                "location ~* /_bulk$ { rewrite /_bulk(.*) /astronomer.$remote_user.*/_bulk$1 break;",
+                "location ~* /_count$ { rewrite /_count(.*) /astronomer.$remote_user.*/_count$1 break;",
+                "location ~* /_search$ { rewrite /_search(.*) /astronomer.$remote_user.*/_search$1 break;",
+                "location = /_cluster/health { proxy_pass http://elasticsearch; }",
+                "location = /_cluster/state/version { proxy_pass http://elasticsearch; }",
+                "location ~ ^/ { deny all; } } }",
+            ]
+        )
+
     def test_elasticsearch_exporter_securitycontext_defaults(self, kube_version):
         """Test ElasticSearch Exporter with securityContext default values."""
         docs = render_chart(

diff --git a/tests/chart_tests/test_external_elasticsearch.py b/tests/chart_tests/test_external_elasticsearch.py
@@ -399,6 +399,88 @@ def test_external_es_index_pattern_defaults(self, kube_version):
         assert doc["kind"] == "ConfigMap"
         assert "fluentd.$remote_user.*/$1" in es_index
 
+    def test_external_es_index_pattern_overrides(self, kube_version):
+        """Test External Elasticsearch Service Index Pattern Search
+        overrides."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={
+                "global": {
+                    "logging": {"indexNamePrefix": "astronomer"},
+                    "customLogging": {
+                        "enabled": True,
+                        "scheme": "https",
+                        "host": "esdemo.example.com",
+                        "awsServiceAccountAnnotation": "arn:aws:iam::xxxxxxxx:role/customrole",
+                    },
+                },
+            },
+            show_only=[
+                "charts/external-es-proxy/templates/external-es-proxy-configmap.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        es_index = doc["data"]["nginx.conf"]
+        assert doc["kind"] == "ConfigMap"
+        assert "astronomer.$remote_user.*/$1" in es_index
+
+    def test_external_es_index_pattern_sidecar_logging_overrides(self, kube_version):
+        """Test External Elasticsearch Service Index Pattern Search
+        overrides."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={
+                "global": {
+                    "logging": {"indexNamePrefix": "astronomer"},
+                    "loggingSidecar": {"enabled": True},
+                    "customLogging": {
+                        "enabled": True,
+                        "scheme": "https",
+                        "host": "esdemo.example.com",
+                        "awsServiceAccountAnnotation": "arn:aws:iam::xxxxxxxx:role/customrole",
+                    },
+                },
+            },
+            show_only=[
+                "charts/external-es-proxy/templates/external-es-proxy-configmap.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        es_index = doc["data"]["nginx.conf"]
+        assert doc["kind"] == "ConfigMap"
+        assert "astronomer.$remote_user.*/$1" in es_index
+
+    def test_external_es_index_pattern_sidecar_logging_defaults(self, kube_version):
+        """Test External Elasticsearch Service Index Pattern Search
+        overrides."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={
+                "global": {
+                    "loggingSidecar": {"enabled": True},
+                    "customLogging": {
+                        "enabled": True,
+                        "scheme": "https",
+                        "host": "esdemo.example.com",
+                        "awsServiceAccountAnnotation": "arn:aws:iam::xxxxxxxx:role/customrole",
+                    },
+                },
+            },
+            show_only=[
+                "charts/external-es-proxy/templates/external-es-proxy-configmap.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        es_index = doc["data"]["nginx.conf"]
+        assert doc["kind"] == "ConfigMap"
+        assert "vector.$remote_user.*/$1" in es_index
+
     def test_external_es_index_pattern_with_sidecar_logging_enabled(self, kube_version):
         """Test External Elasticsearch Service Index Pattern Search with
         sidecar logging."""