Add Python Helm testing framework (apache#11693)

* Helm Python Testing

* helm change

* add back args

.dockerignore

@@ -31,6 +31,7 @@
 !common
 !dags
 !dev
+!chart
 !docs
 !licenses
 !metastore_browser

.github/workflows/ci.yml

@@ -143,7 +143,7 @@ jobs:
 
   helm-tests:
     timeout-minutes: 5
-    name: "Checks: Helm tests"
+    name: "Checks: Helm tests. Will soon be replaced with python tests"
     runs-on: ubuntu-latest
     needs: [build-info]
     if: >
@@ -432,6 +432,52 @@ jobs:
           name: airflow-provider-readmes
           path: "./files/airflow-readme-*"
 
+  helm-python-tests:
+    timeout-minutes: 5
+    name: "Python unit tests for helm chart"
+    runs-on: ubuntu-latest
+    needs: [build-info, ci-images]
+    env:
+      MOUNT_LOCAL_SOURCES: "true"
+      RUN_TESTS: true
+      TEST_TYPES: "Helm"
+      BACKEND: "sqlite"
+      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+    if: >
+      needs.build-info.outputs.needs-helm-tests == 'true' &&
+      (github.repository == 'apache/airflow' || github.event_name != 'schedule')
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      - name: "Free space"
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+      - name: "Upload airflow logs"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: airflow-logs-helm
+          path: "./files/airflow_logs*"
+      - name: "Upload container logs"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: container-logs-helm
+          path: "./files/container_logs*"
+      - name: "Upload artifact for coverage"
+        uses: actions/upload-artifact@v2
+        with:
+          name: >
+            coverage-helm
+          path: "./files/coverage.xml"
+
 
   tests-postgres:
     timeout-minutes: 60
@@ -467,7 +513,7 @@ jobs:
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+      - name: "Tests: Helm"
         run: ./scripts/ci/testing/ci_run_airflow_testing.sh
       - name: "Upload airflow logs"
         uses: actions/upload-artifact@v2

.pre-commit-config.yaml

@@ -159,7 +159,7 @@ repos:
     rev: 20.8b1
     hooks:
       - id: black
-        files: api_connexion/.*\.py|.*providers.*\.py
+        files: api_connexion/.*\.py|.*providers.*\.py|^chart/tests/.*\.py
         exclude: .*kubernetes_pod\.py|.*google/common/hooks/base_google\.py$
         args: [--config=./pyproject.toml]
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -207,7 +207,15 @@ repos:
         args:
           - --convention=pep257
           - --add-ignore=D100,D102,D104,D105,D107,D205,D400,D401
-        exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^provider_packages|^kubernetes_tests|.*example_dags/.*
+        exclude: |
+          (?x)
+          ^tests/.*\.py$|
+          ^scripts/.*\.py$|
+          ^dev|
+          ^provider_packages|
+          ^kubernetes_tests|
+          .*example_dags/.*|
+          ^chart/.*\.py$
   - repo: local
     hooks:
       - id: lint-openapi
@@ -283,7 +291,7 @@ repos:
         entry: "^\\s*from\\s+\\."
         pass_filenames: true
         files: \.py$
-        exclude: ^tests/
+        exclude: ^tests/|^chart/tests/
       - id: language-matters
         language: pygrep
         name: Check for language that we do not accept as community
@@ -301,7 +309,7 @@ repos:
           ^CHANGELOG.txt$
       - id: consistent-pylint
         language: pygrep
-        name: Check for inconsitent pylint disable/enable without space
+        name: Check for inconsistent pylint disable/enable without space
         entry: "pylint:disable|pylint:enable"
         pass_filenames: true
         files: \.py$
@@ -457,6 +465,12 @@ repos:
         language: system
         entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
         files: \.py$
+        exclude: ^dev|^provider_packages|^chart
+      - id: mypy
+        name: Run mypy for helm chart tests
+        language: system
+        entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
+        files: ^chart/.*\.py$
         exclude: ^dev|^provider_packages
         require_serial: true
       - id: pylint

BREEZE.rst

@@ -2003,11 +2003,12 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  consistent-pylint daysago-import-check debug-statements detect-private-key doctoc
                  dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs
                  helm-lint incorrect-use-of-LoggingMixin insert-license isort language-matters
-                 lint-dockerfile lint-openapi mermaid mixed-line-ending mypy no-relative-imports
-                 pre-commit-descriptions provide-create-sessions pydevd pydocstyle pylint
-                 pylint-tests python-no-log-warn restrict-start_date rst-backticks setup-order
-                 shellcheck sort-in-the-wild stylelint trailing-whitespace update-breeze-file
-                 update-extras update-local-yml-file update-setup-cfg-file yamllint
+                 lint-dockerfile lint-openapi mermaid mixed-line-ending mypy mypy-helm
+                 no-relative-imports pre-commit-descriptions provide-create-sessions pydevd
+                 pydocstyle pylint pylint-tests python-no-log-warn restrict-start_date rst-backticks
+                 setup-order shellcheck sort-in-the-wild stylelint trailing-whitespace
+                 update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
+                 yamllint
 
         You can pass extra arguments including options to to the pre-commit framework as
         <EXTRA_ARGS> passed after --. For example:
@@ -2042,7 +2043,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
   --test-type TEST_TYPE
           Type of the test to run. One of:
 
-                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL,Helm
 
           Default: All
 
@@ -2419,7 +2420,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
   --test-type TEST_TYPE
           Type of the test to run. One of:
 
-                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL,Helm
 
           Default: All
 

Dockerfile.ci

@@ -350,6 +350,14 @@ RUN echo "source /etc/bash_completion" >> ~/.bashrc
 
 WORKDIR ${AIRFLOW_SOURCES}
 
+# Install Helm
+ARG HELM_VERSION="v3.2.4"
+
+RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
+    && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \
+    && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
+    && chmod +x /usr/local/bin/helm
+
 # Additional python deps to install
 ARG ADDITIONAL_PYTHON_DEPS=""
 

STATIC_CODE_CHECKS.rst

@@ -118,6 +118,8 @@ require Breeze Docker images to be installed locally:
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``mypy``                              Runs mypy.                                                           *
 ----------------------------------- ---------------------------------------------------------------- ------------
+``mypy-helm``                         Runs mypy.                                                           *
+----------------------------------- ---------------------------------------------------------------- ------------
 ``pre-commit-descriptions``           Check if all pre-commits are described in docs.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``provide-create-sessions``           Make sure provide-session and create-session imports are OK.

TESTING.rst

@@ -196,6 +196,48 @@ Run all Quarantined tests:
 
      ./breeze --test-type Quarantined tests --db-reset
 
+Helm Unit Tests
+===============
+
+On the Airflow Project, we have decided to stick with pythonic testing for our Helm chart. This makes our chart
+easier to test, easier to modify, and able to run with the same testing infrastructure. To add Helm unit tests
+go to the ``chart/tests`` directory and add your unit test by creating a class that extends ``unittest.TestCase``
+
+.. code-block:: python
+
+    class TestBaseChartTest(unittest.TestCase):
+
+To render the chart create a yaml string with the nested dictionary of options you wish to test. you can then
+use our ``render_chart`` function to render the object of interest into a testable python dictionary. Once the chart
+has been rendered, you can use the ``render_k8s_object`` function to create a k8s model object that simultaneously
+ensures that the object created properly conforms to the expected object spec and allows you to use object values
+instead of nested dictionaries.
+
+Example test here:
+
+.. code-block:: python
+    from .helm_template_generator import render_chart, render_k8s_object
+
+
+    git_sync_basic = """
+    dags:
+      gitSync:
+      enabled: true
+    """
+
+    class TestGitSyncScheduler(unittest.TestCase):
+
+    def test_basic(self):
+        helm_settings = yaml.safe_load(git_sync_basic)
+        res = render_chart('GIT-SYNC', helm_settings,
+                           show_only=["templates/scheduler/scheduler-deployment.yaml"])
+        dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+        self.assertEqual("dags", dep.spec.template.spec.volumes[1].name)
+
+To run tests using breeze run the following command
+
+.. code-block:: bash
+    ./breeze --test-type Helm tests
 
 Airflow Integration Tests
 =========================

breeze-complete

@@ -33,7 +33,7 @@ _breeze_allowed_kind_versions="v0.8.0"
 _breeze_allowed_mysql_versions="5.7 8"
 _breeze_allowed_postgres_versions="9.6 10 11 12 13"
 _breeze_allowed_kind_operations="start stop restart status deploy test shell"
-_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL"
+_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL Helm"
 
 # shellcheck disable=SC2034
 {
@@ -100,6 +100,7 @@ lint-openapi
 mermaid
 mixed-line-ending
 mypy
+mypy-helm
 no-relative-imports
 pre-commit-descriptions
 provide-create-sessions

chart/requirements.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com
+  repository: https://kubernetes-charts.storage.googleapis.com/
   version: 6.3.12
-digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af
-generated: "2020-06-21T19:11:53.498134738+02:00"
+digest: sha256:e8d53453861c590e6ae176331634c9268a11cf894be17ed580fa2b347101be97
+generated: "2020-10-27T21:16:13.0063538Z"

chart/tests/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

chart/tests/conftest.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import subprocess
+import sys
+import pytest
+
+# We should set these before loading _any_ of the rest of airflow so that the
+# unit test mode config is set as early as possible.
+tests_directory = os.path.dirname(os.path.realpath(__file__))
+
+
+@pytest.fixture(autouse=True, scope="session")
+def upgrade_helm():
+    """
+    Upgrade Helm repo
+    """
+    subprocess.check_output(
+        ["helm", "repo", "add", "stable", "https://kubernetes-charts.storage.googleapis.com/"]
+    )
+    subprocess.check_output(["helm", "dep", "update", sys.path[0]])