add default index config kibana (#1993)

* add default index config kibana * move template to kibana chart * fix pre-commit * add support for vector pattern * Update charts/kibana/templates/kibana-default-index-cronjob.yaml Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * fix pre-commit * Update charts/kibana/templates/kibana-default-index-cronjob.yaml Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com> * add network policy for index job * add testcases * update kibana index image to ap-init * update job template * fix pull secret template * fix tests * fix pre-commit * fix pre-commit * add functional test for kibana index pod * add namespace * add feature flag for create default index --------- Co-authored-by: Daniel Hoherd <daniel.hoherd@gmail.com>
astronomer · Oct 4, 2023 · c7ad09c · c7ad09c
1 parent 3e83899
commit c7ad09c
Show file tree

Hide file tree

Showing 9 changed files with 164 additions and 2 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -359,6 +359,7 @@ workflows:
                 - quay.io/astronomer/ap-grafana:10.0.2
                 - quay.io/astronomer/ap-houston-api:0.33.4
                 - quay.io/astronomer/ap-init:3.18.3
+                - quay.io/astronomer/ap-init:3.18.3-1
                 - quay.io/astronomer/ap-kibana:8.8.2
                 - quay.io/astronomer/ap-kube-state:2.8.2
                 - quay.io/astronomer/ap-nats-exporter:0.10.0-6

diff --git a/charts/kibana/templates/_helpers.tpl b/charts/kibana/templates/_helpers.tpl
@@ -46,6 +46,14 @@ kibana.{{ .Values.global.baseDomain }}
 {{- end }}
 {{- end }}
 
+{{ define "kibana.init.image" -}}
+{{- if .Values.global.privateRegistry.enabled -}}
+{{ .Values.global.privateRegistry.repository }}/ap-init:{{ .Values.images.init.tag }}
+{{- else -}}
+{{ .Values.images.init.repository }}:{{ .Values.images.init.tag }}
+{{- end }}
+{{- end }}
+
 {{/*
 Return the proper Docker Image Registry Secret Names
 */}}
@@ -55,3 +63,7 @@ imagePullSecrets:
   - name: {{ .Values.global.privateRegistry.secretName }}
 {{- end -}}
 {{- end -}}
+
+{{ define "kibana.IndexPattern" -}}
+{{- if .Values.global.loggingSidecar.enabled }}vector{{- else }}fluentd{{- end -}}
+{{- end }}
diff --git a/charts/kibana/templates/kibana-default-index-cronjob.yaml b/charts/kibana/templates/kibana-default-index-cronjob.yaml
@@ -0,0 +1,49 @@
+########################################
+# Kibana default index helm hook
+########################################
+{{ if .Values.createDefaultIndex }}
+apiVersion: {{ include "apiVersion.batch.cronjob" . }}
+kind: Job
+metadata:
+  name: {{ template "kibana.fullname" . }}-default-index
+  labels:
+    tier: logging
+    component: kibana-default-index
+    chart: {{ template "kibana.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-upgrade,post-install
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+      metadata:
+        labels:
+          tier: logging
+          component: kibana-default-index
+          release: {{ .Release.Name }}
+          app: kibana-default-index
+          version: {{ .Chart.Version }}
+      spec:
+{{- include "kibana.imagePullSecrets" . | indent 8 }}
+        containers:
+        - name: kibana-default-index
+          image: {{ template "kibana.init.image" . }}
+          imagePullPolicy: {{ .Values.images.init.pullPolicy }}
+          resources: {{ toYaml .Values.resources | nindent 12 }}
+          command:
+          - "/bin/sh"
+          - -ec
+          - |
+            http_status=$(curl --retry 10 --retry-delay 30 --retry-all-errors -s -o /dev/null -w "%{http_code}" -XGET http://{{ .Release.Name }}-kibana:5601/api/data_views/data_view/{{ template "kibana.IndexPattern" . }}.*)
+            if [[ "$http_status" -eq 200 ]]; then
+              echo "Kibana Index Pattern Already Exists. Skipping Creation"
+            else
+              echo "Creating Kibana Index Pattern"
+              curl --retry 10 --retry-delay 30 --retry-all-errors -XPOST -H 'Content-Type: application/json' -H 'kbn-xsrf: astronomer' --data '{"attributes":{"title":"{{ template "kibana.IndexPattern" . }}.*","timeFieldName":"@timestamp"}}' http://{{ .Release.Name }}-kibana:5601/api/saved_objects/index-pattern/{{ template "kibana.IndexPattern" . }}.*?overwrite=false
+            fi
+        restartPolicy: Never
+{{ end }}
diff --git a/charts/kibana/templates/kibana-networkpolicy.yaml b/charts/kibana/templates/kibana-networkpolicy.yaml
@@ -33,6 +33,13 @@ spec:
           component: ingress-controller
           release: {{ .Release.Name }}
     {{- end }}
+    {{ if .Values.createDefaultIndex }}
+    - podSelector:
+        matchLabels:
+          component: kibana-default-index
+          release: {{ .Release.Name }}
+          tier: logging
+    {{- end }}
     ports:
     {{- if .Values.global.authSidecar.enabled  }}
     - protocol: TCP

diff --git a/charts/kibana/values.yaml b/charts/kibana/values.yaml
@@ -7,6 +7,10 @@ images:
     repository: quay.io/astronomer/ap-kibana
     tag: 8.8.2
     pullPolicy: IfNotPresent
+  init:
+    repository: quay.io/astronomer/ap-init
+    tag: 3.18.3-1
+    pullPolicy: IfNotPresent
 
 securityContext:
     runAsNonRoot: true
@@ -35,3 +39,5 @@ serviceType: ClusterIP
 
 ports:
   http: 5601
+
+createDefaultIndex: true
diff --git a/tests/chart_tests/test_authsidecar.py b/tests/chart_tests/test_authsidecar.py
@@ -120,7 +120,16 @@ def test_authSidecar_kibana(self, kube_version):
                 "namespaceSelector": {
                     "matchLabels": {"network.openshift.io/policy-group": "ingress"}
                 }
-            }
+            },
+            {
+                "podSelector": {
+                    "matchLabels": {
+                        "component": "kibana-default-index",
+                        "release": "release-name",
+                        "tier": "logging",
+                    }
+                }
+            },
         ] == jmespath.search("spec.ingress[0].from", docs[3])
         assert [{"port": 8084, "protocol": "TCP"}] == jmespath.search(
             "spec.ingress[*].ports[0]", docs[3]

diff --git a/tests/chart_tests/test_kibana.py b/tests/chart_tests/test_kibana.py
@@ -0,0 +1,60 @@
+import pytest
+
+from tests import supported_k8s_versions
+from tests.chart_tests.helm_template_generator import render_chart
+
+
+@pytest.mark.parametrize(
+    "kube_version",
+    supported_k8s_versions,
+)
+class TestKibana:
+    def test_kibana_index_defaults(self, kube_version):
+        """Test kibana Service with index defaults."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={},
+            show_only=[
+                "charts/kibana/templates/kibana-default-index-cronjob.yaml",
+            ],
+        )
+        assert len(docs) == 1
+        doc = docs[0]
+        assert doc["kind"] == "Job"
+        assert doc["apiVersion"] == "batch/v1"
+        assert doc["metadata"]["name"] == "release-name-kibana-default-index"
+        assert (
+            "fluentd.*"
+            in doc["spec"]["template"]["spec"]["containers"][0]["command"][2]
+        )
+
+    def test_kibana_index_with_logging_sidecar(self, kube_version):
+        """Test kibana Service with logging sidecar index."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={"global": {"loggingSidecar": {"enabled": True}}},
+            show_only=[
+                "charts/kibana/templates/kibana-default-index-cronjob.yaml",
+            ],
+        )
+
+        assert len(docs) == 1
+        doc = docs[0]
+        assert doc["kind"] == "Job"
+        assert doc["apiVersion"] == "batch/v1"
+        assert doc["metadata"]["name"] == "release-name-kibana-default-index"
+        assert (
+            "vector.*" in doc["spec"]["template"]["spec"]["containers"][0]["command"][2]
+        )
+
+    def test_kibana_index_disabled(self, kube_version):
+        """Test kibana Service with index creation disabled."""
+        docs = render_chart(
+            kube_version=kube_version,
+            values={"kibana": {"createDefaultIndex": False}},
+            show_only=[
+                "charts/kibana/templates/kibana-default-index-cronjob.yaml",
+            ],
+        )
+
+        assert len(docs) == 0
diff --git a/tests/functional_tests/conftest.py b/tests/functional_tests/conftest.py
@@ -130,3 +130,13 @@ def get_pod_running_containers(pod_namespace=namespace):
                 containers[key] = container
 
     return containers
+
+
+@pytest.fixture(scope="function")
+def kibana_index_pod_client(core_v1_client):
+    pod = get_pod_by_label_selector(
+        core_v1_client, "component=kibana-default-index,tier=logging"
+    )
+    yield testinfra.get_host(
+        f"kubectl://{pod}?container=kibana-default-index&namespace={namespace}"
+    )
diff --git a/tests/functional_tests/test_config.py b/tests/functional_tests/test_config.py
@@ -8,7 +8,7 @@
 import json
 import time
 from os import getenv
-from subprocess import check_output
+from subprocess import check_output, Popen, PIPE
 
 import pytest
 import testinfra
@@ -242,3 +242,11 @@ def test_cve_2021_44228_es_master(es_master):
     assert "-Dlog4j2.formatMsgNoLookups=true" in es_master.check_output(
         "/usr/share/elasticsearch/jdk/bin/jps -lv"
     )
+
+
+def test_kibana_index_pod(kibana_index_pod_client):
+    """Check kibana index pod completed successfully"""
+    command = ["kubectl -n astronomer logs -f  -lcomponent=kibana-default-index"]
+    pod_output = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = pod_output.communicate()
+    assert "fluentd.*" in stdout.decode("utf-8")