Improve handling of default namespace for Helm charts

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Improvements
 
 -   Move YAML decode logic into provider. (https://github.com/pulumi/pulumi-kubernetes/pull/925).
+-   Improve handling of default namespaces for Helm charts. (https://github.com/pulumi/pulumi-kubernetes/pull/934).
 
 ### Bug fixes
 

pkg/clients/clients.go

@@ -163,6 +163,11 @@ func (dcs *DynamicClientSet) IsNamespacedKind(gvk schema.GroupVersionKind) (bool
 		gv = "v1"
 	}
 
+	if known, namespaced := kinds.Kind(gvk.Kind).Namespaced(); known {
+		return namespaced, nil
+	}
+
+	// If the Kind is not known, attempt to look up from the API server. This applies to Kinds defined using a CRD.
 	resourceList, err := dcs.DiscoveryClientCached.ServerResourcesForGroupVersion(gv)
 	if err != nil {
 		return false, &NoNamespaceInfoErr{gvk}

pkg/gen/nodejs-templates/helm/v2/helm.ts

@@ -206,7 +206,8 @@ export class Chart extends yaml.CollectionComponentResource {
                         maxBuffer: 512 * 1024 * 1024 // 512 MB
                     },
                 ).toString();
-                return this.parseTemplate(yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps);
+                return this.parseTemplate(
+                    yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace);
             } catch (e) {
                 // Shed stack trace, only emit the error.
                 throw new pulumi.RunError(e.toString());
@@ -223,9 +224,10 @@ export class Chart extends yaml.CollectionComponentResource {
         transformations: ((o: any, opts: pulumi.CustomResourceOptions) => void)[] | undefined,
         resourcePrefix: string | undefined,
         dependsOn: pulumi.Resource[],
+        defaultNamespace: string | undefined,
     ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> {
         const promise = pulumi.runtime.invoke(
-            "kubernetes:yaml:decode", {text}, {async: true});
+            "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true});
         return pulumi.output(promise).apply(p => yaml.parse(
             {
                 resourcePrefix: resourcePrefix,

pkg/gen/python-templates/helm/v2/helm.py

@@ -348,7 +348,8 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi
 
     chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd)
     objects = chart_resources.apply(
-        lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'])
+        lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', {
+            'text': text, 'defaultNamespace': config.namespace}).value['result'])
 
     # Parse the manifest and create the specified resources.
     resources = objects.apply(
@@ -473,7 +474,7 @@ def get_resource(self, group_version_kind, name, namespace=None) -> pulumi.Outpu
 
         # `id` will either be `${name}` or `${namespace}/${name}`.
         id = pulumi.Output.from_input(name)
-        if namespace != None:
+        if namespace is not None:
             id = pulumi.Output.concat(namespace, '/', name)
 
         resource_id = id.apply(lambda x: f'{group_version_kind}:{x}')

pkg/kinds/kinds.go

@@ -18,24 +18,31 @@ type Kind string
 
 const (
 	APIService                     Kind = "APIService"
+	Binding                        Kind = "Binding"
 	CertificateSigningRequest      Kind = "CertificateSigningRequest"
 	ClusterRole                    Kind = "ClusterRole"
 	ClusterRoleBinding             Kind = "ClusterRoleBinding"
+	ComponentStatus                Kind = "ComponentStatus"
 	ControllerRevision             Kind = "ControllerRevision"
 	CustomResourceDefinition       Kind = "CustomResourceDefinition"
 	ConfigMap                      Kind = "ConfigMap"
 	CronJob                        Kind = "CronJob"
+	CSIDriver                      Kind = "CSIDriver"
+	CSINode                        Kind = "CSINode"
 	DaemonSet                      Kind = "DaemonSet"
 	Deployment                     Kind = "Deployment"
 	Endpoints                      Kind = "Endpoints"
 	Event                          Kind = "Event"
 	HorizontalPodAutoscaler        Kind = "HorizontalPodAutoscaler"
 	Ingress                        Kind = "Ingress"
 	Job                            Kind = "Job"
+	Lease                          Kind = "Lease"
 	LimitRange                     Kind = "LimitRange"
+	LocalSubjectAccessReview       Kind = "LocalSubjectAccessReview"
 	MutatingWebhookConfiguration   Kind = "MutatingWebhookConfiguration"
 	Namespace                      Kind = "Namespace"
 	NetworkPolicy                  Kind = "NetworkPolicy"
+	Node                           Kind = "Node"
 	PersistentVolume               Kind = "PersistentVolume"
 	PersistentVolumeClaim          Kind = "PersistentVolumeClaim"
 	Pod                            Kind = "Pod"
@@ -48,10 +55,82 @@ const (
 	ResourceQuota                  Kind = "ResourceQuota"
 	Role                           Kind = "Role"
 	RoleBinding                    Kind = "RoleBinding"
+	RuntimeClass                   Kind = "RuntimeClass"
 	Secret                         Kind = "Secret"
+	SelfSubjectAccessReview        Kind = "SelfSubjectAccessReview"
+	SelfSubjectRulesReview         Kind = "SelfSubjectRulesReview"
 	Service                        Kind = "Service"
 	ServiceAccount                 Kind = "ServiceAccount"
 	StatefulSet                    Kind = "StatefulSet"
+	SubjectAccessReview            Kind = "SubjectAccessReview"
 	StorageClass                   Kind = "StorageClass"
+	TokenReview                    Kind = "TokenReview"
 	ValidatingWebhookConfiguration Kind = "ValidatingWebhookConfiguration"
+	VolumeAttachment               Kind = "VolumeAttachment"
 )
+
+// Namespaced returns whether known resource Kinds are namespaced. If the Kind is unknown (such as CRD Kinds), the
+// known return value will be false, and the namespaced value is unknown. In this case, this information can be
+// queried separately from the k8s API server.
+func (k Kind) Namespaced() (known bool, namespaced bool) {
+	switch k {
+	// Note: Use `kubectl api-resources --namespaced=true -o name` to retrieve a list of namespace-scoped resources.
+	case Binding,
+		ConfigMap,
+		ControllerRevision,
+		CronJob,
+		DaemonSet,
+		Deployment,
+		Endpoints,
+		Event,
+		HorizontalPodAutoscaler,
+		Ingress,
+		Job,
+		Lease,
+		LimitRange,
+		LocalSubjectAccessReview,
+		NetworkPolicy,
+		PersistentVolumeClaim,
+		Pod,
+		PodDisruptionBudget,
+		PodTemplate,
+		ReplicaSet,
+		ReplicationController,
+		ResourceQuota,
+		Role,
+		RoleBinding,
+		Secret,
+		Service,
+		ServiceAccount,
+		StatefulSet:
+		known, namespaced = true, true
+	// Note: Use `kubectl api-resources --namespaced=false -o name` to retrieve a list of cluster-scoped resources.
+	case APIService,
+		CertificateSigningRequest,
+		ClusterRole,
+		ClusterRoleBinding,
+		ComponentStatus,
+		CSIDriver,
+		CSINode,
+		CustomResourceDefinition,
+		MutatingWebhookConfiguration,
+		Namespace,
+		Node,
+		PersistentVolume,
+		PodSecurityPolicy,
+		PriorityClass,
+		RuntimeClass,
+		SelfSubjectAccessReview,
+		SelfSubjectRulesReview,
+		StorageClass,
+		SubjectAccessReview,
+		TokenReview,
+		ValidatingWebhookConfiguration,
+		VolumeAttachment:
+		known, namespaced = true, false
+	default:
+		known = false
+	}
+
+	return
+}

pkg/provider/invoke_decode_yaml.go

@@ -19,13 +19,14 @@ import (
 	"io/ioutil"
 	"strings"
 
+	"github.com/pulumi/pulumi-kubernetes/pkg/clients"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
 // decodeYaml parses a YAML string, and then returns a slice of untyped structs that can be marshalled into
-// Pulumi RPC calls.
-func decodeYaml(text string) ([]interface{}, error) {
+// Pulumi RPC calls. If a default namespace is specified, set that on the relevant decoded objects.
+func decodeYaml(text, defaultNamespace string, clientSet *clients.DynamicClientSet) ([]interface{}, error) {
 	var resources []unstructured.Unstructured
 
 	dec := yaml.NewYAMLOrJSONDecoder(ioutil.NopCloser(strings.NewReader(text)), 128)
@@ -37,7 +38,30 @@ func decodeYaml(text string) ([]interface{}, error) {
 			}
 			return nil, err
 		}
-		resources = append(resources, unstructured.Unstructured{Object: value})
+		resource := unstructured.Unstructured{Object: value}
+
+		// Sometimes manifests include empty resources, so skip these.
+		if len(resource.GetKind()) == 0 || len(resource.GetAPIVersion()) == 0 {
+			continue
+		}
+
+		if len(defaultNamespace) > 0 {
+			namespaced, err := clientSet.IsNamespacedKind(resource.GroupVersionKind())
+			if err != nil {
+				if clients.IsNoNamespaceInfoErr(err) {
+					// Assume resource is namespaced.
+					namespaced = true
+				} else {
+					return nil, err
+				}
+			}
+
+			// Set namespace if resource Kind is namespaced and namespace is not already set.
+			if namespaced && len(resource.GetNamespace()) == 0 {
+				resource.SetNamespace(defaultNamespace)
+			}
+		}
+		resources = append(resources, resource)
 	}
 
 	result := make([]interface{}, len(resources))

pkg/provider/provider.go

@@ -387,14 +387,17 @@ func (k *kubeProvider) Invoke(ctx context.Context,
 
 	switch tok {
 	case invokeDecodeYaml:
-		var text string
+		var text, defaultNamespace string
 		if args["text"].HasValue() {
 			text = args["text"].StringValue()
 		} else {
 			return nil, fmt.Errorf("missing required field 'text' on input: %#v", args)
 		}
+		if args["defaultNamespace"].HasValue() {
+			defaultNamespace = args["defaultNamespace"].StringValue()
+		}
 
-		result, err := decodeYaml(text)
+		result, err := decodeYaml(text, defaultNamespace, k.clientSet)
 		if err != nil {
 			return nil, err
 		}

sdk/nodejs/helm/v2/helm.ts

@@ -206,7 +206,8 @@ export class Chart extends yaml.CollectionComponentResource {
                         maxBuffer: 512 * 1024 * 1024 // 512 MB
                     },
                 ).toString();
-                return this.parseTemplate(yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps);
+                return this.parseTemplate(
+                    yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace);
             } catch (e) {
                 // Shed stack trace, only emit the error.
                 throw new pulumi.RunError(e.toString());
@@ -223,9 +224,10 @@ export class Chart extends yaml.CollectionComponentResource {
         transformations: ((o: any, opts: pulumi.CustomResourceOptions) => void)[] | undefined,
         resourcePrefix: string | undefined,
         dependsOn: pulumi.Resource[],
+        defaultNamespace: string | undefined,
     ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> {
         const promise = pulumi.runtime.invoke(
-            "kubernetes:yaml:decode", {text}, {async: true});
+            "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true});
         return pulumi.output(promise).apply(p => yaml.parse(
             {
                 resourcePrefix: resourcePrefix,

sdk/python/pulumi_kubernetes/helm/v2/helm.py

@@ -348,7 +348,8 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi
 
     chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd)
     objects = chart_resources.apply(
-        lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'])
+        lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', {
+            'text': text, 'defaultNamespace': config.namespace}).value['result'])
 
     # Parse the manifest and create the specified resources.
     resources = objects.apply(
@@ -473,7 +474,7 @@ def get_resource(self, group_version_kind, name, namespace=None) -> pulumi.Outpu
 
         # `id` will either be `${name}` or `${namespace}/${name}`.
         id = pulumi.Output.from_input(name)
-        if namespace != None:
+        if namespace is not None:
             id = pulumi.Output.concat(namespace, '/', name)
 
         resource_id = id.apply(lambda x: f'{group_version_kind}:{x}')

tests/examples/helm-local/index.ts

@@ -23,6 +23,7 @@ function chart(resourcePrefix?: string): k8s.helm.v2.Chart {
         // Represents chart `stable/nginx-lego@v0.3.1`.
         path: "nginx-lego",
         version: "0.3.1",
+        namespace: namespaceName,
         resourcePrefix: resourcePrefix,
         values: {
             // Override for the Chart's `values.yml` file. Use `null` to zero out resource requests so it
@@ -42,14 +43,6 @@ function chart(resourcePrefix?: string): k8s.helm.v2.Chart {
                     }
                 }
             },
-            // Put every resource in the created namespace.
-            (obj: any) => {
-                if (obj.metadata !== undefined) {
-                    obj.metadata.namespace = namespaceName;
-                } else {
-                    obj.metadata = {namespace: namespaceName};
-                }
-            }
         ]
     });
 }

tests/examples/helm/index.ts

@@ -23,6 +23,7 @@ const nginx = new k8s.helm.v2.Chart("simple-nginx", {
     repo: "stable",
     chart: "nginx-lego",
     version: "0.3.1",
+    namespace: namespaceName,
     values: {
         // Override for the Chart's `values.yml` file. Use `null` to zero out resource requests so it
         // can be scheduled on our (wimpy) CI cluster. (Setting these values to `null` is the "normal"
@@ -41,14 +42,6 @@ const nginx = new k8s.helm.v2.Chart("simple-nginx", {
                 }
             }
         },
-        // Put every resource in the created namespace.
-        (obj: any) => {
-            if (obj.metadata !== undefined) {
-                obj.metadata.namespace = namespaceName
-            } else {
-                obj.metadata = {namespace: namespaceName}
-            }
-        }, 
         (obj: any, opts: pulumi.CustomResourceOptions) => {
             if (obj.kind == "Service" && obj.apiVersion == "v1") {
                 opts.additionalSecretOutputs = ["status"];

tests/integration/istio/step1/index.ts

@@ -16,7 +16,7 @@ import * as k8s from "@pulumi/kubernetes";
 import * as pulumi from "@pulumi/pulumi";
 
 import { k8sProvider } from "./cluster";
-import { istio } from "./istio";
+import { crd10, crd11, crd12, istio } from "./istio";
 
 new k8s.core.v1.Namespace(
     "bookinfo",
@@ -33,13 +33,13 @@ function addNamespace(o: any) {
 const bookinfo = new k8s.yaml.ConfigFile(
     "yaml/bookinfo.yaml",
     { transformations: [addNamespace] },
-    { dependsOn: istio, providers: { kubernetes: k8sProvider } }
+    { dependsOn: [crd10, crd11, crd12], providers: { kubernetes: k8sProvider } }
 );
 
 new k8s.yaml.ConfigFile(
     "yaml/bookinfo-gateway.yaml",
     { transformations: [addNamespace] },
-    { dependsOn: bookinfo, providers: { kubernetes: k8sProvider } }
+    { dependsOn: [crd10, crd11, crd12], providers: { kubernetes: k8sProvider } }
 );
 
 export const port = istio

tests/integration/istio/step1/istio.ts

@@ -52,9 +52,9 @@ export const istio_init = new k8s.helm.v2.Chart(
     { dependsOn: [namespace, adminBinding], providers: { kubernetes: k8sProvider } }
 );
 
-const crd10 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-10");
-const crd11 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-11");
-const crd12 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-12");
+export const crd10 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-10");
+export const crd11 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-11");
+export const crd12 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-12");
 
 export const istio = new k8s.helm.v2.Chart(
     appName,