diff --git a/chart/tests/conftest.py b/chart/tests/conftest.py index 789f31174135d..7ac27d9be9607 100644 --- a/chart/tests/conftest.py +++ b/chart/tests/conftest.py @@ -19,12 +19,29 @@ import sys import pytest +from filelock import FileLock @pytest.fixture(autouse=True, scope="session") -def upgrade_helm(): +def upgrade_helm(tmp_path_factory, worker_id): """ Upgrade Helm repo """ - subprocess.check_output(["helm", "repo", "add", "stable", "https://charts.helm.sh/stable/"]) - subprocess.check_output(["helm", "dep", "update", sys.path[0]]) + + def _upgrade_helm(): + subprocess.check_output(["helm", "repo", "add", "stable", "https://charts.helm.sh/stable/"]) + subprocess.check_output(["helm", "dep", "update", sys.path[0]]) + + if worker_id == "main": + # not executing in with multiple workers, just update + _upgrade_helm() + return + + root_tmp_dir = tmp_path_factory.getbasetemp().parent + lock_fn = root_tmp_dir / "upgrade_helm.lock" + flag_fn = root_tmp_dir / "upgrade_helm.done" + + with FileLock(str(lock_fn)): + if not flag_fn.is_file(): + _upgrade_helm() + flag_fn.touch() diff --git a/chart/tests/helm_template_generator.py b/chart/tests/helm_template_generator.py index 56a0691f26241..6523d32895d14 100644 --- a/chart/tests/helm_template_generator.py +++ b/chart/tests/helm_template_generator.py @@ -83,16 +83,17 @@ def validate_k8s_object(instance): validate.validate(instance) -def render_chart(name="RELEASE-NAME", values=None, show_only=None): +def render_chart(name="RELEASE-NAME", values=None, show_only=None, chart_dir=None): """ Function that renders a helm chart into dictionaries. For helm chart testing only """ values = values or {} + chart_dir = chart_dir or sys.path[0] with NamedTemporaryFile() as tmp_file: content = yaml.dump(values) tmp_file.write(content.encode()) tmp_file.flush() - command = ["helm", "template", name, sys.path[0], '--values', tmp_file.name] + command = ["helm", "template", name, chart_dir, '--values', tmp_file.name] if show_only: for i in show_only: command.extend(["--show-only", i]) diff --git a/chart/tests/test_pod_template_file.py b/chart/tests/test_pod_template_file.py index 2a315e4d17023..d949ec161cb1f 100644 --- a/chart/tests/test_pod_template_file.py +++ b/chart/tests/test_pod_template_file.py @@ -16,33 +16,36 @@ # under the License. import re +import sys import unittest -from os import remove -from os.path import dirname, realpath -from shutil import copyfile +from shutil import copyfile, copytree +from tempfile import TemporaryDirectory import jmespath +import pytest from parameterized import parameterized from tests.helm_template_generator import render_chart -ROOT_FOLDER = realpath(dirname(realpath(__file__)) + "/..") - class PodTemplateFileTest(unittest.TestCase): - def setUp(self): - copyfile( - ROOT_FOLDER + "/files/pod-template-file.kubernetes-helm-yaml", - ROOT_FOLDER + "/templates/pod-template-file.yaml", - ) - - def tearDown(self): - remove(ROOT_FOLDER + "/templates/pod-template-file.yaml") + @classmethod + @pytest.fixture(autouse=True, scope="class") + def isolate_chart(cls): + with TemporaryDirectory() as tmp_dir: + cls.temp_chart_dir = tmp_dir + "/chart" + copytree(sys.path[0], cls.temp_chart_dir) + copyfile( + cls.temp_chart_dir + "/files/pod-template-file.kubernetes-helm-yaml", + cls.temp_chart_dir + "/templates/pod-template-file.yaml", + ) + yield def test_should_work(self): docs = render_chart( values={}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -79,6 +82,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): }, }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -111,6 +115,7 @@ def test_should_not_add_init_container_if_dag_persistence_is_true(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.initContainers", docs[0]) is None @@ -131,6 +136,7 @@ def test_dags_mount(self, dag_values): docs = render_chart( values={"dags": dag_values}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search( @@ -146,6 +152,7 @@ def test_dags_mount_with_gitsync_and_persistence(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search( @@ -166,6 +173,7 @@ def test_validate_if_ssh_params_are_added(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( @@ -196,6 +204,7 @@ def test_validate_if_ssh_known_hosts_are_added(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "GIT_KNOWN_HOSTS", "value": "true"} in jmespath.search( "spec.initContainers[0].env", docs[0] @@ -222,6 +231,7 @@ def test_should_set_username_and_pass_env_variables(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert { @@ -237,6 +247,7 @@ def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim docs = render_chart( values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "dags", "persistentVolumeClaim": {"claimName": "test-claim"}} in jmespath.search( @@ -247,6 +258,7 @@ def test_should_use_empty_dir_for_gitsync_without_persistence(self): docs = render_chart( values={"dags": {"gitSync": {"enabled": True}}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "dags", "emptyDir": {}} in jmespath.search("spec.volumes", docs[0]) @@ -265,6 +277,7 @@ def test_logs_persistence_changes_volume(self, log_persistence_values, expected) docs = render_chart( values={"logs": {"persistence": log_persistence_values}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "logs", **expected} in jmespath.search("spec.volumes", docs[0]) @@ -273,6 +286,7 @@ def test_should_set_a_custom_image_in_pod_template(self): docs = render_chart( values={"images": {"pod_template": {"repository": "dummy_image", "tag": "latest"}}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -283,6 +297,7 @@ def test_mount_airflow_cfg(self): docs = render_chart( values={}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -318,6 +333,7 @@ def test_should_create_valid_affinity_and_node_selector(self): "nodeSelector": {"diskType": "ssd"}, }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -342,6 +358,7 @@ def test_should_add_fsgroup_to_the_pod_template(self): docs = render_chart( values={"gid": 5000}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) self.assertEqual(5000, jmespath.search("spec.securityContext.fsGroup", docs[0])) @@ -355,6 +372,7 @@ def test_should_create_valid_volume_mount_and_volume(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert "test-volume" == jmespath.search( diff --git a/setup.py b/setup.py index 9331d1c6b9b0a..6083c14aa59de 100644 --- a/setup.py +++ b/setup.py @@ -484,6 +484,7 @@ def get_sphinx_theme_version() -> str: 'click~=7.1', 'coverage', 'docutils', + 'filelock', 'flake8>=3.6.0', 'flake8-colors', 'flaky',