Fix error message assertions in Helm tests (#7858)

helm/dagster/schema/schema/utils/helm_template.py

@@ -47,19 +47,21 @@ def render(
                 self.name,
                 helm_dir_path,
                 "--debug",
-                *["--values", tmp_file.name],
+                "--values",
+                tmp_file.name,
             ]
 
-            if self.output:
-                ## Uncomment to render all templates before filtering to surface Helm templating
-                ## errors with better error messages
-                # subprocess.check_output(command)
-
-                command += ["--show-only", self.output]
-
             with self._with_chart_yaml(helm_dir_path, chart_version):
                 templates = subprocess.check_output(command)
 
+                # HACK! Helm's --show-only option doesn't surface errors. For tests where we want to
+                # assert on things like {{ fail ... }}, we need to render the chart without --show-only.
+                # If that succeeds, we then carry on to calling with --show-only so that we can
+                # assert on specific objects in the chart.
+                if self.output:
+                    command += ["--show-only", self.output]
+                    templates = subprocess.check_output(command)
+
             print("\n--- Helm Templates ---")  # pylint: disable=print-call
             print(templates.decode())  # pylint: disable=print-call
 

helm/dagster/schema/schema_tests/test_instance_migrate.py

@@ -17,14 +17,14 @@ def helm_template() -> HelmTemplate:
     )
 
 
-def test_job_instance_migrate_does_not_render(template: HelmTemplate, capsys):
+def test_job_instance_migrate_does_not_render(template: HelmTemplate, capfd):
     with pytest.raises(subprocess.CalledProcessError):
         helm_values_migrate_disabled = DagsterHelmValues.construct(migrate=Migrate(enabled=False))
 
         template.render(helm_values_migrate_disabled)
 
-        _, err = capsys.readouterr()
-        assert "Error: could not find template" in err
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
 
 
 def test_job_instance_migrate_renders(template: HelmTemplate):

helm/dagster/schema/schema_tests/test_service_account.py

@@ -69,18 +69,17 @@ def test_service_account_global_name(template: HelmTemplate):
     assert service_account_template.metadata.name == global_service_account_name
 
 
-def test_subchart_service_account_global_name(subchart_template: HelmTemplate, capsys):
-    with pytest.raises(subprocess.CalledProcessError):
-        global_service_account_name = "global-service-account-name"
-        service_account_values = DagsterHelmValues.construct(
-            global_=Global.construct(serviceAccountName=global_service_account_name),
-        )
+def test_subchart_service_account_global_name(subchart_template: HelmTemplate, capfd):
+    global_service_account_name = "global-service-account-name"
+    service_account_values = DagsterHelmValues.construct(
+        global_=Global.construct(serviceAccountName=global_service_account_name),
+    )
 
+    with pytest.raises(subprocess.CalledProcessError):
         subchart_template.render(service_account_values)
 
-        _, err = capsys.readouterr()
-
-        assert "Error: could not find template" in err
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
 
 
 def test_standalone_subchart_service_account_name(standalone_subchart_template: HelmTemplate):
@@ -98,17 +97,15 @@ def test_standalone_subchart_service_account_name(standalone_subchart_template:
     assert service_account_template.metadata.name == service_account_name
 
 
-def test_service_account_does_not_render(template: HelmTemplate, capsys):
+def test_service_account_does_not_render(template: HelmTemplate, capfd):
+    service_account_values = DagsterHelmValues.construct(
+        serviceAccount=ServiceAccount.construct(name="service-account-name", create=False),
+    )
     with pytest.raises(subprocess.CalledProcessError):
-        service_account_values = DagsterHelmValues.construct(
-            serviceAccount=ServiceAccount.construct(name="service-account-name", create=False),
-        )
-
         template.render(service_account_values)
 
-        _, err = capsys.readouterr()
-
-        assert "Error: could not find template" in err
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
 
 
 def test_service_account_annotations(template: HelmTemplate):

helm/dagster/schema/schema_tests/test_user_deployments.py

@@ -142,43 +142,55 @@ def assert_user_deployment_template(
             assert template_resources == resource_values
 
 
-@pytest.mark.parametrize(
-    "helm_values",
-    [
-        DagsterHelmValues.construct(
-            dagsterUserDeployments=UserDeployments(
-                enabled=False,
-                enableSubchart=False,
-                deployments=[create_simple_user_deployment("simple-deployment-one")],
-            )
-        ),
-        DagsterHelmValues.construct(
-            dagsterUserDeployments=UserDeployments(
-                enabled=False,
-                enableSubchart=True,
-                deployments=[create_simple_user_deployment("simple-deployment-one")],
-            )
-        ),
-        DagsterHelmValues.construct(
-            dagsterUserDeployments=UserDeployments(
-                enabled=True,
-                enableSubchart=False,
-                deployments=[create_simple_user_deployment("simple-deployment-one")],
-            )
-        ),
-    ],
-    ids=[
-        "user deployments disabled, subchart disabled",
-        "user deployments disabled, subchart enabled",
-        "user deployments enabled, subchart disabled",
-    ],
-)
-def test_deployments_do_not_render(helm_values: DagsterHelmValues, template: HelmTemplate, capsys):
+def test_deployments_enabled_subchart_disabled(template: HelmTemplate, capfd):
     with pytest.raises(subprocess.CalledProcessError):
-        template.render(helm_values)
+        template.render(
+            DagsterHelmValues.construct(
+                dagsterUserDeployments=UserDeployments(
+                    enabled=True,
+                    enableSubchart=False,
+                    deployments=[create_simple_user_deployment("simple-deployment-one")],
+                )
+            ),
+        )
 
-        _, err = capsys.readouterr()
-        assert "Error: could not find template" in err
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
+
+
+def test_deployments_disabled_subchart_enabled(template: HelmTemplate, capfd):
+    with pytest.raises(subprocess.CalledProcessError):
+        template.render(
+            DagsterHelmValues.construct(
+                dagsterUserDeployments=UserDeployments(
+                    enabled=False,
+                    enableSubchart=True,
+                    deployments=[create_simple_user_deployment("simple-deployment-one")],
+                )
+            ),
+        )
+
+    _, err = capfd.readouterr()
+    assert (
+        "dagster-user-deployments subchart cannot be enabled if workspace.yaml is not created"
+        in err
+    )
+
+
+def test_deployments_disabled_subchart_disabled(template: HelmTemplate, capfd):
+    with pytest.raises(subprocess.CalledProcessError):
+        template.render(
+            DagsterHelmValues.construct(
+                dagsterUserDeployments=UserDeployments(
+                    enabled=False,
+                    enableSubchart=False,
+                    deployments=[create_simple_user_deployment("simple-deployment-one")],
+                )
+            )
+        )
+
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
 
 
 @pytest.mark.parametrize(
@@ -233,7 +245,7 @@ def test_deployments_render(helm_values: DagsterHelmValues, template: HelmTempla
     assert_user_deployment_template(template, user_deployments, helm_values)
 
 
-def test_chart_does_not_render(full_template: HelmTemplate, capsys):
+def test_chart_does_not_render(full_template: HelmTemplate, capfd):
     helm_values = DagsterHelmValues.construct(
         dagsterUserDeployments=UserDeployments(
             enabled=False,
@@ -245,11 +257,11 @@ def test_chart_does_not_render(full_template: HelmTemplate, capsys):
     with pytest.raises(subprocess.CalledProcessError):
         full_template.render(helm_values)
 
-        _, err = capsys.readouterr()
-        assert (
-            "dagster-user-deployments subchart cannot be enabled if workspace.yaml is not created."
-            in err
-        )
+    _, err = capfd.readouterr()
+    assert (
+        "dagster-user-deployments subchart cannot be enabled if workspace.yaml is not created."
+        in err
+    )
 
 
 @pytest.mark.parametrize(

helm/dagster/schema/schema_tests/test_workspace.py

@@ -22,7 +22,7 @@ def helm_template() -> HelmTemplate:
     )
 
 
-def test_workspace_renders_fail(template: HelmTemplate, capsys):
+def test_workspace_renders_fail(template: HelmTemplate, capfd):
     helm_values = DagsterHelmValues.construct(
         dagsterUserDeployments=UserDeployments(
             enabled=False,
@@ -34,14 +34,14 @@ def test_workspace_renders_fail(template: HelmTemplate, capsys):
     with pytest.raises(subprocess.CalledProcessError):
         template.render(helm_values)
 
-        _, err = capsys.readouterr()
-        assert (
-            "dagster-user-deployments subchart cannot be enabled if workspace.yaml is not created."
-            in err
-        )
+    _, err = capfd.readouterr()
+    assert (
+        "dagster-user-deployments subchart cannot be enabled if workspace.yaml is not created"
+        in err
+    )
 
 
-def test_workspace_does_not_render(template: HelmTemplate, capsys):
+def test_workspace_does_not_render(template: HelmTemplate, capfd):
     helm_values = DagsterHelmValues.construct(
         dagsterUserDeployments=UserDeployments(
             enabled=False,
@@ -53,8 +53,8 @@ def test_workspace_does_not_render(template: HelmTemplate, capsys):
     with pytest.raises(subprocess.CalledProcessError):
         template.render(helm_values)
 
-        _, err = capsys.readouterr()
-        assert "Error: could not find template" in err in err
+    _, err = capfd.readouterr()
+    assert "Error: could not find template" in err
 
 
 def test_workspace_renders_from_helm_user_deployments(template: HelmTemplate):