Added skipUpdateUnreachable flag (#2528)

The Pulumi ```deleteUnreachable``` flag is great when a cluster is
removed but there is no way to proceed with applying the changes on
multiple K8s clusters if there is just one not available, in this case
the full update will fail (.e.g updating charts over multiple K8s
clusters where 1+ are not currently available).

The proposed feature introduce a new variable called
**```skipUpdateUnreachable```** and allow to run ```pulumi up``` without
failing if one or more Kubernetes cluster are unreachable (showing
warning messages).
The feature allows to make changes on an existing system which might
consist of multiple K8 clusters and at that specific moment has some
clusters down.

Co-authored-by: Gianmarco <gianmarco.ascenzo@infovista.com>
Co-authored-by: Ramon Quitales <ramon@pulumi.com>

CHANGELOG.md

@@ -4,9 +4,9 @@
   source-based and wheel distributions are now published. For most users the installs will now favor
   the wheel distribution, but users invoking pip with `--no-binary :all:` will continue having
   installs based on the source distribution.
-
 - Return mapping information for terraform conversions (https://github.com/pulumi/pulumi-kubernetes/pull/2457)
-
+- feature: added skipUpdateUnreachable flag to proceed with the updates without failing (https://github.com/pulumi/pulumi-kubernetes/pull/2528)
+- 
 ## 4.1.1 (August 23, 2023)
 
 - Revert the switch to pyproject.toml and wheel-based PyPI publishing as it impacts users that run pip with --no-binary

provider/cmd/pulumi-resource-kubernetes/schema.json

@@ -496,6 +496,10 @@
                 "type": "string",
                 "description": "BETA FEATURE - If present, render resource manifests to this directory. In this mode, resources will not\nbe created on a Kubernetes cluster, but the rendered manifests will be kept in sync with changes\nto the Pulumi program. This feature is in developer preview, and is disabled by default.\n\nNote that some computed Outputs such as status fields will not be populated\nsince the resources are not created on a Kubernetes cluster. These Output values will remain undefined,\nand may result in an error if they are referenced by other resources. Also note that any secret values\nused in these resources will be rendered in plaintext to the resulting YAML."
             },
+            "skipUpdateUnreachable": {
+                "type": "boolean",
+                "description": "If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state"
+            },
             "strictMode": {
                 "type": "boolean",
                 "description": "If present and set to true, the provider will use strict configuration mode. Recommended for production stacks. In this mode, the default Kubernetes provider is disabled, and the `kubeconfig` and `context` settings are required for Provider configuration. These settings unambiguously ensure that every Kubernetes resource is associated with a particular cluster."
@@ -65083,6 +65087,15 @@
                 "type": "string",
                 "description": "BETA FEATURE - If present, render resource manifests to this directory. In this mode, resources will not\nbe created on a Kubernetes cluster, but the rendered manifests will be kept in sync with changes\nto the Pulumi program. This feature is in developer preview, and is disabled by default.\n\nNote that some computed Outputs such as status fields will not be populated\nsince the resources are not created on a Kubernetes cluster. These Output values will remain undefined,\nand may result in an error if they are referenced by other resources. Also note that any secret values\nused in these resources will be rendered in plaintext to the resulting YAML."
             },
+            "skipUpdateUnreachable": {
+                "type": "boolean",
+                "description": "If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state",
+                "defaultInfo": {
+                    "environment": [
+                        "PULUMI_K8S_SKIP_UPDATE_UNREACHABLE"
+                    ]
+                }
+            },
             "suppressDeprecationWarnings": {
                 "type": "boolean",
                 "description": "If present and set to true, suppress apiVersion deprecation warnings from the CLI.",

provider/pkg/gen/schema.go

@@ -73,6 +73,10 @@ func PulumiSchema(swagger map[string]any) pschema.PackageSpec {
 					Description: "If present and set to true, the provider will delete resources associated with an unreachable Kubernetes cluster from Pulumi state",
 					TypeSpec:    pschema.TypeSpec{Type: "boolean"},
 				},
+				"skipUpdateUnreachable": {
+					Description: "If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state",
+					TypeSpec:    pschema.TypeSpec{Type: "boolean"},
+				},
 				"enableServerSideApply": {
 					Description: "If present and set to false, disable Server-Side Apply mode.\nSee https://github.com/pulumi/pulumi-kubernetes/issues/2011 for additional details.",
 					TypeSpec:    pschema.TypeSpec{Type: "boolean"},
@@ -142,6 +146,15 @@ func PulumiSchema(swagger map[string]any) pschema.PackageSpec {
 						},
 					},
 				},
+				"skipUpdateUnreachable": {
+					Description: "If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state",
+					TypeSpec:    pschema.TypeSpec{Type: "boolean"},
+					DefaultInfo: &pschema.DefaultSpec{
+						Environment: []string{
+							"PULUMI_K8S_SKIP_UPDATE_UNREACHABLE",
+						},
+					},
+				},
 				"namespace": {
 					Description: "If present, the default namespace to use. This flag is ignored for cluster-scoped resources.\n\nA namespace can be specified in multiple places, and the precedence is as follows:\n1. `.metadata.namespace` set on the resource.\n2. This `namespace` parameter.\n3. `namespace` set for the active context in the kubeconfig.",
 					TypeSpec:    pschema.TypeSpec{Type: "string"},

provider/pkg/provider/provider.go

@@ -125,6 +125,7 @@ type kubeProvider struct {
 	defaultNamespace string
 
 	deleteUnreachable           bool
+	skipUpdateUnreachable       bool
 	enableConfigMapMutable      bool
 	enableSecrets               bool
 	suppressDeprecationWarnings bool
@@ -170,6 +171,7 @@ func makeKubeProvider(
 		enableSecrets:               false,
 		suppressDeprecationWarnings: false,
 		deleteUnreachable:           false,
+		skipUpdateUnreachable:       false,
 	}, nil
 }
 
@@ -470,6 +472,22 @@ func (k *kubeProvider) Configure(_ context.Context, req *pulumirpc.ConfigureRequ
 		k.deleteUnreachable = true
 	}
 
+	skipUpdateUnreachable := func() bool {
+		// If the provider flag is set, use that value to determine behavior. This will override the ENV var.
+		if enabled, exists := vars["kubernetes:config:skipUpdateUnreachable"]; exists {
+			return enabled == trueStr
+		}
+		// If the provider flag is not set, fall back to the ENV var.
+		if enabled, exists := os.LookupEnv("PULUMI_K8S_SKIP_UPDATE_UNREACHABLE"); exists {
+			return enabled == trueStr
+		}
+		// Default to false.
+		return false
+	}
+	if skipUpdateUnreachable() {
+		k.skipUpdateUnreachable = true
+	}
+
 	enableServerSideApply := func() bool {
 		// If the provider flag is set, use that value to determine behavior. This will override the ENV var.
 		if enabled, exists := vars["kubernetes:config:enableServerSideApply"]; exists {
@@ -1262,6 +1280,16 @@ func (k *kubeProvider) Check(ctx context.Context, req *pulumirpc.CheckRequest) (
 	//
 
 	urn := resource.URN(req.GetUrn())
+
+	if k.clusterUnreachable {
+		if k.skipUpdateUnreachable {
+			_ = k.host.Log(ctx, diag.Warning, urn, "Cluster is unreachable but skipUpdateUnreachable flag is set to true, skipping...")
+			return &pulumirpc.CheckResponse{
+				Inputs: req.GetOlds(),
+			}, nil
+		}
+	}
+
 	if isHelmRelease(urn) {
 		return k.helmReleaseProvider.Check(ctx, req)
 	}
@@ -1931,6 +1959,16 @@ func (k *kubeProvider) Read(ctx context.Context, req *pulumirpc.ReadRequest) (*p
 					"Deleting the unreachable resource from Pulumi state"))
 			return deleteResponse, nil
 		}
+		if k.skipUpdateUnreachable {
+			_ = k.host.Log(ctx, diag.Info, urn, fmt.Sprintf(
+				"configured Kubernetes cluster is unreachable and the `skipUnreachable` option is enabled. "+
+					"Returned data could not reflect the actual cluster configuration."))
+			return &pulumirpc.ReadResponse{
+				Id:         req.GetId(),
+				Properties: req.GetProperties(),
+				Inputs:     req.GetInputs(),
+			}, nil
+		}
 
 		return nil, fmt.Errorf("failed to read resource state due to unreachable cluster. If the cluster was " +
 			"deleted, you can remove this resource from Pulumi state by rerunning the operation with the " +

sdk/dotnet/Config/Config.cs

@@ -140,6 +140,16 @@ public static string? RenderYamlToDirectory
             set => _renderYamlToDirectory.Set(value);
         }
 
+        private static readonly __Value<bool?> _skipUpdateUnreachable = new __Value<bool?>(() => __config.GetBoolean("skipUpdateUnreachable"));
+        /// <summary>
+        /// If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+        /// </summary>
+        public static bool? SkipUpdateUnreachable
+        {
+            get => _skipUpdateUnreachable.Get();
+            set => _skipUpdateUnreachable.Set(value);
+        }
+
         private static readonly __Value<bool?> _strictMode = new __Value<bool?>(() => __config.GetBoolean("strictMode"));
         /// <summary>
         /// If present and set to true, the provider will use strict configuration mode. Recommended for production stacks. In this mode, the default Kubernetes provider is disabled, and the `kubeconfig` and `context` settings are required for Provider configuration. These settings unambiguously ensure that every Kubernetes resource is associated with a particular cluster.

sdk/dotnet/Provider.cs

@@ -124,6 +124,12 @@ public sealed class ProviderArgs : global::Pulumi.ResourceArgs
         [Input("renderYamlToDirectory")]
         public Input<string>? RenderYamlToDirectory { get; set; }
 
+        /// <summary>
+        /// If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+        /// </summary>
+        [Input("skipUpdateUnreachable", json: true)]
+        public Input<bool>? SkipUpdateUnreachable { get; set; }
+
         /// <summary>
         /// If present and set to true, suppress apiVersion deprecation warnings from the CLI.
         /// </summary>
@@ -142,6 +148,7 @@ public ProviderArgs()
             EnableConfigMapMutable = Utilities.GetEnvBoolean("PULUMI_K8S_ENABLE_CONFIGMAP_MUTABLE");
             EnableServerSideApply = Utilities.GetEnvBoolean("PULUMI_K8S_ENABLE_SERVER_SIDE_APPLY");
             KubeConfig = Utilities.GetEnv("KUBECONFIG");
+            SkipUpdateUnreachable = Utilities.GetEnvBoolean("PULUMI_K8S_SKIP_UPDATE_UNREACHABLE");
             SuppressDeprecationWarnings = Utilities.GetEnvBoolean("PULUMI_K8S_SUPPRESS_DEPRECATION_WARNINGS");
             SuppressHelmHookWarnings = Utilities.GetEnvBoolean("PULUMI_K8S_SUPPRESS_HELM_HOOK_WARNINGS");
         }

sdk/java/src/main/java/com/pulumi/kubernetes/Config.java

@@ -92,6 +92,13 @@ public Optional<String> namespace() {
     public Optional<String> renderYamlToDirectory() {
         return Codegen.stringProp("renderYamlToDirectory").config(config).get();
     }
+/**
+ * If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+ * 
+ */
+    public Optional<Boolean> skipUpdateUnreachable() {
+        return Codegen.booleanProp("skipUpdateUnreachable").config(config).get();
+    }
 /**
  * If present and set to true, the provider will use strict configuration mode. Recommended for production stacks. In this mode, the default Kubernetes provider is disabled, and the `kubeconfig` and `context` settings are required for Provider configuration. These settings unambiguously ensure that every Kubernetes resource is associated with a particular cluster.
  * 

sdk/java/src/main/java/com/pulumi/kubernetes/ProviderArgs.java

@@ -205,6 +205,21 @@ public Optional<Output<String>> renderYamlToDirectory() {
         return Optional.ofNullable(this.renderYamlToDirectory);
     }
 
+    /**
+     * If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+     * 
+     */
+    @Import(name="skipUpdateUnreachable", json=true)
+    private @Nullable Output<Boolean> skipUpdateUnreachable;
+
+    /**
+     * @return If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+     * 
+     */
+    public Optional<Output<Boolean>> skipUpdateUnreachable() {
+        return Optional.ofNullable(this.skipUpdateUnreachable);
+    }
+
     /**
      * If present and set to true, suppress apiVersion deprecation warnings from the CLI.
      * 
@@ -248,6 +263,7 @@ private ProviderArgs(ProviderArgs $) {
         this.kubeconfig = $.kubeconfig;
         this.namespace = $.namespace;
         this.renderYamlToDirectory = $.renderYamlToDirectory;
+        this.skipUpdateUnreachable = $.skipUpdateUnreachable;
         this.suppressDeprecationWarnings = $.suppressDeprecationWarnings;
         this.suppressHelmHookWarnings = $.suppressHelmHookWarnings;
     }
@@ -516,6 +532,27 @@ public Builder renderYamlToDirectory(String renderYamlToDirectory) {
             return renderYamlToDirectory(Output.of(renderYamlToDirectory));
         }
 
+        /**
+         * @param skipUpdateUnreachable If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+         * 
+         * @return builder
+         * 
+         */
+        public Builder skipUpdateUnreachable(@Nullable Output<Boolean> skipUpdateUnreachable) {
+            $.skipUpdateUnreachable = skipUpdateUnreachable;
+            return this;
+        }
+
+        /**
+         * @param skipUpdateUnreachable If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+         * 
+         * @return builder
+         * 
+         */
+        public Builder skipUpdateUnreachable(Boolean skipUpdateUnreachable) {
+            return skipUpdateUnreachable(Output.of(skipUpdateUnreachable));
+        }
+
         /**
          * @param suppressDeprecationWarnings If present and set to true, suppress apiVersion deprecation warnings from the CLI.
          * 
@@ -563,6 +600,7 @@ public ProviderArgs build() {
             $.enableConfigMapMutable = Codegen.booleanProp("enableConfigMapMutable").output().arg($.enableConfigMapMutable).env("PULUMI_K8S_ENABLE_CONFIGMAP_MUTABLE").getNullable();
             $.enableServerSideApply = Codegen.booleanProp("enableServerSideApply").output().arg($.enableServerSideApply).env("PULUMI_K8S_ENABLE_SERVER_SIDE_APPLY").getNullable();
             $.kubeconfig = Codegen.stringProp("kubeconfig").output().arg($.kubeconfig).env("KUBECONFIG").getNullable();
+            $.skipUpdateUnreachable = Codegen.booleanProp("skipUpdateUnreachable").output().arg($.skipUpdateUnreachable).env("PULUMI_K8S_SKIP_UPDATE_UNREACHABLE").getNullable();
             $.suppressDeprecationWarnings = Codegen.booleanProp("suppressDeprecationWarnings").output().arg($.suppressDeprecationWarnings).env("PULUMI_K8S_SUPPRESS_DEPRECATION_WARNINGS").getNullable();
             $.suppressHelmHookWarnings = Codegen.booleanProp("suppressHelmHookWarnings").output().arg($.suppressHelmHookWarnings).env("PULUMI_K8S_SUPPRESS_HELM_HOOK_WARNINGS").getNullable();
             return $;

sdk/nodejs/provider.ts

@@ -47,6 +47,7 @@ export class Provider extends pulumi.ProviderResource {
             resourceInputs["kubeconfig"] = (args ? args.kubeconfig : undefined) ?? utilities.getEnv("KUBECONFIG");
             resourceInputs["namespace"] = args ? args.namespace : undefined;
             resourceInputs["renderYamlToDirectory"] = args ? args.renderYamlToDirectory : undefined;
+            resourceInputs["skipUpdateUnreachable"] = pulumi.output((args ? args.skipUpdateUnreachable : undefined) ?? utilities.getEnvBoolean("PULUMI_K8S_SKIP_UPDATE_UNREACHABLE")).apply(JSON.stringify);
             resourceInputs["suppressDeprecationWarnings"] = pulumi.output((args ? args.suppressDeprecationWarnings : undefined) ?? utilities.getEnvBoolean("PULUMI_K8S_SUPPRESS_DEPRECATION_WARNINGS")).apply(JSON.stringify);
             resourceInputs["suppressHelmHookWarnings"] = pulumi.output((args ? args.suppressHelmHookWarnings : undefined) ?? utilities.getEnvBoolean("PULUMI_K8S_SUPPRESS_HELM_HOOK_WARNINGS")).apply(JSON.stringify);
         }
@@ -117,6 +118,10 @@ export interface ProviderArgs {
      * used in these resources will be rendered in plaintext to the resulting YAML.
      */
     renderYamlToDirectory?: pulumi.Input<string>;
+    /**
+     * If present and set to true, the provider will skip resources update associated with an unreachable Kubernetes cluster from Pulumi state
+     */
+    skipUpdateUnreachable?: pulumi.Input<boolean>;
     /**
      * If present and set to true, suppress apiVersion deprecation warnings from the CLI.
      */