Fix: Make the invoke calls for Helm charts and YAML config resilient …

…to the value being `None` or an empty dict (#2665)

### Proposed changes

In v3.92.0 of the core `pulumi` Python SDK, invokes were changed to
return an empty dict rather than `None` for empty results, which is
consistent with other language SDKs and the original behavior of the
Python SDK, and makes generated provider Python SDKs work with empty
results. However, the hand-rolled invoke code in the Kubernetes Python
SDK for Helm and YAML are not resilient to it getting an empty dict.
This change fixes Helm and YAML to be resilient, in the same way that
the invoke for Kustomize was updated in
a907bdd.

An invoke will return empty results when called on an unconfigured
provider (i.e. a provider whose configuration contains an unknown
value). The first three commits add regression tests that pass an
unknown value as a provider's configuration. The Helm and YAML
regression tests fail before the fix and succeed after the fix. The test
for Kustomize already succeeds because of the change in
a907bdd. The new tests are based on
existing tests.

To ensure older versions of `pulumi-kubernetes` continue to work with
newer versions of `pulumi`, we're planning to patch the `pulumi` SDK to
return `None` for empty invoke returns when the function is one of the
affected kubernetes function tokens. This is tracked by
pulumi/pulumi#14508 (PR:
pulumi/pulumi#14535).

### Related issues

Fixes #2664

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Unreleased
 
+- Fix: Make the invoke calls for Helm charts and YAML config resilient to the value being None or an empty dict (https://github.com/pulumi/pulumi-kubernetes/pull/2665)
+
 ## 4.5.4 (November 8, 2023)
 - Fix: Helm Release: chart requires kubeVersion (https://github.com/pulumi/pulumi-kubernetes/pull/2653)
 

provider/pkg/gen/python-templates/helm/v3/helm.py

@@ -619,6 +619,6 @@ def _parse_chart(all_config: Tuple[Union[ChartOpts, LocalChartOpts], pulumi.Reso
 
     def invoke_helm_template(opts):
         inv = pulumi.runtime.invoke('kubernetes:helm:template', {'jsonOpts': opts}, invoke_opts)
-        return inv.value['result'] if inv is not None and inv.value is not None else []
+        return (inv.value or {}).get('result', [])
     objects = json_opts.apply(invoke_helm_template)
     return objects.apply(lambda x: _parse_yaml_document(x, opts, transformations))

provider/pkg/gen/python-templates/yaml/yaml.tmpl

@@ -517,4 +517,4 @@ def _parse_yaml_object(
 
 def invoke_yaml_decode(text, invoke_opts):
     inv = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}, invoke_opts)
-    return inv.value['result'] if inv is not None and inv.value is not None else []
+    return (inv.value or {}).get('result', [])

sdk/python/pulumi_kubernetes/helm/v3/helm.py

@@ -619,6 +619,6 @@ def _parse_chart(all_config: Tuple[Union[ChartOpts, LocalChartOpts], pulumi.Reso
 
     def invoke_helm_template(opts):
         inv = pulumi.runtime.invoke('kubernetes:helm:template', {'jsonOpts': opts}, invoke_opts)
-        return inv.value['result'] if inv is not None and inv.value is not None else []
+        return (inv.value or {}).get('result', [])
     objects = json_opts.apply(invoke_helm_template)
     return objects.apply(lambda x: _parse_yaml_document(x, opts, transformations))

sdk/python/pulumi_kubernetes/yaml/yaml.py

@@ -1949,4 +1949,4 @@ def _parse_yaml_object(
 
 def invoke_yaml_decode(text, invoke_opts):
     inv = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}, invoke_opts)
-    return inv.value['result'] if inv is not None and inv.value is not None else []
+    return (inv.value or {}).get('result', [])

tests/sdk/python/helm-local-unconfigured-provider/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: helm-local-unconfigured-provider
+description: A program that tests Helm chart creation from a local directory with an unconfigured provider
+runtime: python

tests/sdk/python/helm-local-unconfigured-provider/__main__.py

@@ -0,0 +1,56 @@
+#  Copyright 2016-2023, Pulumi Corporation.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Any
+
+from pulumi_kubernetes import Provider
+from pulumi_kubernetes.core.v1 import Namespace
+from pulumi_kubernetes.helm.v3 import Chart, LocalChartOpts
+from pulumi import Config, ResourceOptions
+
+
+def set_namespace(namespace):
+    def f(obj: Any):
+        if "metadata" in obj:
+            obj["metadata"]["namespace"] = namespace.metadata["name"]
+        else:
+            obj["metadata"] = {"namespace": namespace.metadata["name"]}
+
+    return f
+
+
+config = Config()
+path = config.require("path")
+
+# This will be unknown during the initial preview.
+unknown = Provider("provider").id.apply(lambda _: True)
+
+# This provider will be unconfigured when the passed-in configuration has an unknown value.
+provider = Provider("k8s", suppress_deprecation_warnings=unknown)
+
+values = {"service": {"type": "ClusterIP"}}
+
+ns = Namespace("unconfiguredtest", opts=ResourceOptions(provider=provider))
+
+# An error shouldn't be raised when called using the unconfigured provider.
+chart = Chart(
+    "nginx",
+    LocalChartOpts(
+        path=path,
+        namespace=ns.metadata.name,
+        values=values,
+        transformations=[set_namespace(ns)],
+    ),
+    opts=ResourceOptions(provider=provider)
+)

tests/sdk/python/helm-local-unconfigured-provider/requirements.txt

@@ -0,0 +1 @@
+pulumi>=3.0.0,<4.0.0

tests/sdk/python/kustomize-unconfigured-provider/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: kustomize-test-unconfigured-provider
+description: Test kustomize support using an unconfigured provider
+runtime: python

tests/sdk/python/kustomize-unconfigured-provider/__main__.py

@@ -0,0 +1,45 @@
+# Copyright 2016-2023, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pulumi_kubernetes as k8s
+from pulumi import ResourceOptions
+
+from typing import Any
+
+
+def set_namespace(namespace):
+    def f(obj: Any):
+        if "metadata" in obj:
+            obj["metadata"]["namespace"] = namespace.metadata["name"]
+        else:
+            obj["metadata"] = {"namespace": namespace.metadata["name"]}
+
+    return f
+
+
+# This will be unknown during the initial preview.
+unknown = k8s.Provider("provider").id.apply(lambda _: True)
+
+# This provider will be unconfigured when the passed-in configuration has an unknown value.
+provider = k8s.Provider("k8s", suppress_deprecation_warnings=unknown)
+
+ns = k8s.core.v1.Namespace("unconfiguredtest", opts=ResourceOptions(provider=provider))
+
+# An error shouldn't be raised when called using the unconfigured provider.
+k8s.kustomize.Directory(
+    "kustomize-local",
+    "helloWorld",
+    transformations=[set_namespace(ns)],
+    opts=ResourceOptions(provider=provider),
+)

tests/sdk/python/kustomize-unconfigured-provider/helloWorld/configMap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: the-map
+data:
+  altGreeting: "Good Morning!"
+  enableRisky: "false"

tests/sdk/python/kustomize-unconfigured-provider/helloWorld/deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: the-deployment
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      deployment: hello
+  template:
+    metadata:
+      labels:
+        deployment: hello
+    spec:
+      containers:
+      - name: the-container
+        image: monopole/hello:1
+        command: ["/hello",
+                  "--port=8080",
+                  "--enableRiskyFeature=$(ENABLE_RISKY)"]
+        ports:
+        - containerPort: 8080
+        env:
+        - name: ALT_GREETING
+          valueFrom:
+            configMapKeyRef:
+              name: the-map
+              key: altGreeting
+        - name: ENABLE_RISKY
+          valueFrom:
+            configMapKeyRef:
+              name: the-map
+              key: enableRisky

tests/sdk/python/kustomize-unconfigured-provider/helloWorld/kustomization.yaml

@@ -0,0 +1,9 @@
+# Example configuration for the webserver
+# at https://github.com/monopole/hello
+commonLabels:
+  app: hello
+
+resources:
+- deployment.yaml
+- service.yaml
+- configMap.yaml

tests/sdk/python/kustomize-unconfigured-provider/helloWorld/service.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: the-service
+spec:
+  selector:
+    deployment: hello
+  type: LoadBalancer
+  ports:
+  - protocol: TCP
+    port: 8666
+    targetPort: 8080

tests/sdk/python/kustomize-unconfigured-provider/requirements.txt

@@ -0,0 +1 @@
+pulumi>=3.0.0,<4.0.0

tests/sdk/python/python_test.go

@@ -231,6 +231,28 @@ func TestYaml(t *testing.T) {
 	integration.ProgramTest(t, &options)
 }
 
+// Regression Test for https://github.com/pulumi/pulumi-kubernetes/issues/2664.
+// Ensure the program runs without an error being raised when an invoke is called
+// using a provider that is not configured.
+func TestYamlUnconfiguredProvider(t *testing.T) {
+	cwd, err := os.Getwd()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	options := baseOptions.With(integration.ProgramTestOptions{
+		Dir:                  filepath.Join(cwd, "yaml-test-unconfigured-provider"),
+		ExpectRefreshChanges: true,
+		OrderedConfig: []integration.ConfigValue{
+			{
+				Key:   "pulumi:disable-default-providers[0]",
+				Value: "kubernetes",
+				Path:  true,
+			},
+		},
+	})
+	integration.ProgramTest(t, &options)
+}
+
 func TestGuestbook(t *testing.T) {
 	cwd, err := os.Getwd()
 	if !assert.NoError(t, err) {
@@ -417,6 +439,32 @@ func TestHelmLocal(t *testing.T) {
 	integration.ProgramTest(t, &options)
 }
 
+// Regression Test for https://github.com/pulumi/pulumi-kubernetes/issues/2664.
+// Ensure the program runs without an error being raised when an invoke is called
+// using a provider that is not configured.
+func TestHelmLocalUnconfiguredProvider(t *testing.T) {
+	cwd, err := os.Getwd()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	options := baseOptions.With(integration.ProgramTestOptions{
+		Dir: filepath.Join(cwd, "helm-local-unconfigured-provider"),
+		OrderedConfig: []integration.ConfigValue{
+			{
+				Key:   "pulumi:disable-default-providers[0]",
+				Value: "kubernetes",
+				Path:  true,
+			},
+			{
+				Key:   "path",
+				Value: filepath.Join(cwd, "..", "..", "testdata", "helm", "nginx"),
+			},
+		},
+		ExpectRefreshChanges: true,
+	})
+	integration.ProgramTest(t, &options)
+}
+
 func TestHelmApiVersions(t *testing.T) {
 	cwd, err := os.Getwd()
 	if !assert.NoError(t, err) {
@@ -484,6 +532,27 @@ func TestKustomize(t *testing.T) {
 	integration.ProgramTest(t, &options)
 }
 
+// Regression Test for https://github.com/pulumi/pulumi-kubernetes/issues/2664.
+// Ensure the program runs without an error being raised when an invoke is called
+// using a provider that is not configured.
+func TestKustomizeUnconfiguredProvider(t *testing.T) {
+	cwd, err := os.Getwd()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	options := baseOptions.With(integration.ProgramTestOptions{
+		Dir: filepath.Join(cwd, "kustomize-unconfigured-provider"),
+		OrderedConfig: []integration.ConfigValue{
+			{
+				Key:   "pulumi:disable-default-providers[0]",
+				Value: "kubernetes",
+				Path:  true,
+			},
+		},
+	})
+	integration.ProgramTest(t, &options)
+}
+
 func TestSecrets(t *testing.T) {
 	cwd, err := os.Getwd()
 	if !assert.NoError(t, err) {

tests/sdk/python/yaml-test-unconfigured-provider/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: yaml-test-unconfigured-provider
+description: A program that tests YAML functionality in the Python SDK with an unconfigured provider
+runtime: python

tests/sdk/python/yaml-test-unconfigured-provider/__main__.py

@@ -0,0 +1,66 @@
+# Copyright 2016-2023, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pulumi_kubernetes import Provider
+from pulumi_kubernetes.core.v1 import Namespace
+from pulumi_kubernetes.yaml import ConfigFile, ConfigGroup
+from pulumi import ResourceOptions
+
+
+def set_namespace(namespace):
+    def f(obj):
+        if "metadata" in obj:
+            obj["metadata"]["namespace"] = namespace.metadata["name"]
+        else:
+            obj["metadata"] = {"namespace": namespace.metadata["name"]}
+
+    return f
+
+
+def secret_status(obj, opts):
+    if obj["kind"] == "Pod" and obj["apiVersion"] == "v1":
+        opts.additional_secret_outputs = ["apiVersion"]
+
+
+# This will be unknown during the initial preview.
+unknown = Provider("provider").id.apply(lambda _: True)
+
+# This provider will be unconfigured when the passed-in configuration has an unknown value.
+provider = Provider("k8s", suppress_deprecation_warnings=unknown)
+
+ns = Namespace("unconfiguredtest", opts=ResourceOptions(provider=provider))
+
+# An error shouldn't be raised when called using the unconfigured provider.
+cf_local = ConfigFile(
+    "yaml-test",
+    "manifest.yaml",
+    transformations=[
+        set_namespace(ns),
+        secret_status,
+    ],
+    opts=ResourceOptions(provider=provider),
+)
+
+# An error shouldn't be raised when called using the unconfigured provider.
+cg = ConfigGroup(
+    "deployment",
+    files=["ns*.yaml"],
+    yaml=["""
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: utcg3
+    """],
+    opts=ResourceOptions(provider=provider)
+)