All k8s object must comply with JSON Schema (#12003)

* All k8s resources should have global labels

* All k8s object must comply with JSON Schema

(cherry picked from commit 3c85c2c)

.pre-commit-config.yaml

@@ -272,14 +272,14 @@ repos:
                 ^\s*def\s*\S*\([^:#)]*:.*|          # Matches function param with Python3 type
                 ^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.*  # Matches -> return value syntax from Python3
             )$
-        exclude: ^dev|^scripts|^docs
+        exclude: ^dev|^scripts|^docs|^chart
         pass_filenames: true
       - id: python2-compile
         name: Compile code using python2
         language: system
         entry: python2.7 -m py_compile
         files: \.py$
-        exclude: ^dev|^scripts|^docs
+        exclude: ^dev|^scripts|^docs|^chart
         pass_filenames: true
         require_serial: true
       - id: incorrect-use-of-LoggingMixin

chart/files/pod-template-file.kubernetes-helm-yaml

@@ -40,12 +40,7 @@ spec:
       volumeMounts:
         - mountPath: {{ template "airflow_logs" . }}
           name: airflow-logs
-{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
-        - mountPath: {{ template "airflow_dags_mount_path" . }}
-          name: airflow-dags
-          readOnly: false
-{{- end }}
-{{- if .Values.dags.gitSync.sshKeySecret }}
+{{- if .Values.dags.gitSync.knownHosts }}
         - mountPath: /etc/git-secret/known_hosts
           name: {{ .Values.dags.gitSync.knownHosts }}
           subPath: known_hosts
@@ -77,7 +72,7 @@ spec:
     {{ toYaml .Values.affinity | indent 8 }}
   tolerations:
     {{ toYaml .Values.tolerations | indent 8 }}
-  serviceAccountName: '{{ .Release.Name }}-worker-serviceaccount'
+  serviceAccountName: '{{ .Release.Name }}-worker'
   volumes:
   {{- if .Values.dags.persistence.enabled }}
   - name: dags

chart/templates/redis/redis-statefulset.yaml

@@ -48,10 +48,10 @@ spec:
 {{- with .Values.labels }}
 {{ toYaml . | indent 8 }}
 {{- end }}
+      {{- if .Values.redis.safeToEvict }}
       annotations:
-        {{- if .Values.redis.safeToEvict }}
         cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
-        {{- end }}
+      {{- end }}
     spec:
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}

chart/templates/scheduler/scheduler-serviceaccount.yaml

@@ -28,13 +28,13 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.scheduler.serviceAccountAnnotations }}
   annotations:
     {{- range $key, $value := . }}
       {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
       {{- end }}
   {{- end }}
-{{- with .Values.labels }}
-{{ toYaml . | indent 4 }}
-{{- end }}
 {{- end }}

chart/templates/secrets/redis-secrets.yaml

@@ -51,6 +51,9 @@ metadata:
     release: {{ .Release.Name }}
     chart: {{ .Chart.Name }}
     heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
   annotations:
     "helm.sh/hook": "pre-install"
     "helm.sh/hook-delete-policy": "before-hook-creation"

chart/templates/webserver/webserver-ingress.yaml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################
+## Airflow Webserver Ingress
+#################################
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: {{ .Release.Name }}-airflow-ingress
+  labels:
+    tier: airflow
+    component: airflow-ingress
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+  {{- with .Values.ingress.web.annotations }}
+  annotations:
+  {{ toYaml . | indent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.web.tls.enabled }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.web.host }}
+      secretName: {{ .Values.ingress.web.tls.secretName }}
+  {{- end }}
+  rules:
+    - http:
+        paths:
+          {{- range .Values.ingress.web.precedingPaths }}
+          - path: {{ .path }}
+            backend:
+              serviceName: {{ .serviceName }}
+              servicePort: {{ .servicePort }}
+          {{- end }}
+          - path: {{ .Values.ingress.web.path }}
+            backend:
+              serviceName: {{ .Release.Name }}-webserver
+              servicePort: airflow-ui
+          {{- range .Values.ingress.web.succeedingPaths }}
+          - path: {{ .path }}
+            backend:
+              serviceName: {{ .serviceName }}
+              servicePort: {{ .servicePort }}
+          {{- end }}
+      host: {{ .Values.ingress.web.host }}
+{{- end }}

chart/templates/webserver/webserver-service.yaml

@@ -31,6 +31,10 @@ metadata:
 {{- with .Values.labels }}
 {{ toYaml . | indent 4 }}
 {{- end }}
+{{- with .Values.webserver.service.annotations }}
+  annotations:
+{{- toYaml . | nindent 4 }}
+{{- end }}
 spec:
   type: {{ .Values.webserver.service.type }}
   selector:

chart/templates/webserver/webserver-serviceaccount.yaml

@@ -27,12 +27,10 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.webserver.serviceAccountAnnotations }}
   annotations:
-    {{- range $key, $value := . }}
-      {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
-      {{- end }}
+  {{ toYaml . | nindent 4 }}
   {{- end }}
-{{- with .Values.labels }}
-{{ toYaml . | indent 4 }}
-{{- end }}

chart/templates/workers/worker-serviceaccount.yaml

@@ -28,13 +28,11 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
-  {{- with .Values.workers.serviceAccountAnnotations }}
+    {{- with .Values.labels }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.workers.serviceAccountAnnotations}}
   annotations:
-    {{- range $key, $value := . }}
-      {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
-      {{- end }}
+    {{ toYaml . | nindent 4 }}
   {{- end }}
-  {{- with .Values.labels }}
-{{ toYaml . | indent 4 }}
-{{- end }}
 {{- end }}

chart/tests/helm_template_generator.py

@@ -17,13 +17,48 @@
 
 import subprocess
 import sys
+from functools import lru_cache
 from tempfile import NamedTemporaryFile
 
+import jmespath
+import jsonschema
+import requests
 import yaml
 from kubernetes.client.api_client import ApiClient
 
 api_client = ApiClient()
 
+BASE_URL_SPEC = "https://raw.githubusercontent.com/instrumenta/kubernetes-json-schema/master/v1.12.9"
+
+
+@lru_cache(maxsize=None)
+def create_validator(api_version, kind):
+    api_version = api_version.lower()
+    kind = kind.lower()
+
+    if '/' in api_version:
+        ext, _, api_version = api_version.partition("/")
+        ext = ext.split(".")[0]
+        url = f'{BASE_URL_SPEC}/{kind}-{ext}-{api_version}.json'
+    else:
+        url = f'{BASE_URL_SPEC}/{kind}-{api_version}.json'
+    request = requests.get(url)
+    request.raise_for_status()
+    schema = request.json()
+    jsonschema.Draft7Validator.check_schema(schema)
+    validator = jsonschema.Draft7Validator(schema)
+    return validator
+
+
+def validate_k8s_object(instance):
+    # Skip PostgresSQL chart
+    chart = jmespath.search("metadata.labels.chart", instance)
+    if chart and 'postgresql' in chart:
+        return
+
+    validate = create_validator(instance.get("apiVersion"), instance.get("kind"))
+    validate.validate(instance)
+
 
 def render_chart(name="RELEASE-NAME", values=None, show_only=None):
     """
@@ -41,6 +76,8 @@ def render_chart(name="RELEASE-NAME", values=None, show_only=None):
         templates = subprocess.check_output(command)
         k8s_objects = yaml.load_all(templates)
         k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object]  # type: ignore
+        for k8s_object in k8s_objects:
+            validate_k8s_object(k8s_object)
         return k8s_objects
 
 

chart/tests/test_basic_helm_chart.py

@@ -17,14 +17,24 @@
 
 import unittest
 
+import jmespath
+
 from tests.helm_template_generator import render_chart
 
 OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 22
 
 
 class TestBaseChartTest(unittest.TestCase):
     def test_basic_deployments(self):
-        k8s_objects = render_chart("TEST-BASIC", {"chart": {'metadata': 'AA'}})
+        k8s_objects = render_chart(
+            "TEST-BASIC",
+            values={
+                "chart": {
+                    'metadata': 'AA',
+                },
+                'labels': {"TEST-LABEL": "TEST-VALUE"},
+            },
+        )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         ]
@@ -56,6 +66,16 @@ def test_basic_deployments(self):
             ],
         )
         self.assertEqual(OBJECT_COUNT_IN_BASIC_DEPLOYMENT, len(k8s_objects))
+        for k8s_object in k8s_objects:
+            labels = jmespath.search('metadata.labels', k8s_object) or {}
+            if 'postgresql' in labels.get('chart'):
+                continue
+            k8s_name = k8s_object['kind'] + ":" + k8s_object['metadata']['name']
+            self.assertEqual(
+                'TEST-VALUE',
+                labels.get("TEST-LABEL"),
+                f"Missing label TEST-LABEL on {k8s_name}. Current labels: {labels}",
+            )
 
     def test_basic_deployment_without_default_users(self):
         k8s_objects = render_chart("TEST-BASIC", {"webserver": {'defaultUser': {'enabled': False}}})