Skip to content

Commit

Permalink
Drop usage of last-applied-configuration annotation
Browse files Browse the repository at this point in the history 
Previous versions of the provider used the "kubectl.kubernetes.io/last-applied-configuration" annotation to store a copy of the configuration as part of the Kubernetes resource. This annotation was used for input diff computation in client-side apply mode. Specifically, input diffs were computed between the new inputs and the last-applied-configuration value from the previous live state. Using this annotation resulted in a large number of reported issues (30+) for the provider, and it will no longer be used.

14916d3 added logic to prune a map based on a target map. This logic is used to prune live state to match the previous inputs before performing an input diff computation. This avoids the need to store the last-applied-configuration.
  • Loading branch information
@lblackstone
lblackstone committed Jun 6, 2023
1 parent 4aed225 commit d91155e
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 135 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Breaking changes:
- Drop support for Kubernetes clusters older than v1.13 (https://github.com/pulumi/pulumi-kubernetes/pull/2414)
- Make all resource output properties required (https://github.com/pulumi/pulumi-kubernetes/pull/2422)
- Drop support for legacy pulumi.com/initialApiVersion annotation (https://github.com/pulumi/pulumi-kubernetes/pull/2443)
- Drop usage of last-applied-configuration annotation (https://github.com/pulumi/pulumi-kubernetes/pull/2445)

Other changes:

Expand Down
171 changes: 36 additions & 135 deletions provider/pkg/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1301,6 +1301,8 @@ func (k *kubeProvider) Check(ctx context.Context, req *pulumirpc.CheckRequest) (

k.helmHookWarning(ctx, newInputs, urn)

newInputs = dropLastAppliedConfigAnnotation(newInputs)

// Adopt name from old object if appropriate.
//
// If the user HAS NOT assigned a name in the new inputs, we autoname it and mark the object as
Expand Down Expand Up @@ -1758,29 +1760,21 @@ func (k *kubeProvider) Create(
return &pulumirpc.CreateResponse{Id: "", Properties: req.GetProperties()}, nil
}

annotatedInputs, err := k.withLastAppliedConfig(newInputs)
if err != nil {
return nil, pkgerrors.Wrapf(
err, "Failed to create resource %s/%s because of an error generating the %s value in "+
"`.metadata.annotations`",
newInputs.GetNamespace(), newInputs.GetName(), lastAppliedConfigKey)
}

initialAPIVersion := newInputs.GetAPIVersion()
fieldManager := k.fieldManagerName(nil, newResInputs, newInputs)

if k.yamlRenderMode {
if newResInputs.ContainsSecrets() {
_ = k.host.Log(ctx, diag.Warning, urn, fmt.Sprintf(
"rendered file %s contains a secret value in plaintext",
renderPathForResource(annotatedInputs, k.yamlDirectory)))
renderPathForResource(newInputs, k.yamlDirectory)))
}
err := renderYaml(annotatedInputs, k.yamlDirectory)
err := renderYaml(newInputs, k.yamlDirectory)
if err != nil {
return nil, err
}

obj := checkpointObject(newInputs, annotatedInputs, newResInputs, initialAPIVersion, fieldManager)
obj := checkpointObject(newInputs, newInputs, newResInputs, initialAPIVersion, fieldManager)
inputsAndComputed, err := plugin.MarshalProperties(
obj, plugin.MarshalOptions{
Label: fmt.Sprintf("%s.inputsAndComputed", label),
Expand All @@ -1793,10 +1787,10 @@ func (k *kubeProvider) Create(
}

_ = k.host.LogStatus(ctx, diag.Info, urn, fmt.Sprintf(
"rendered %s", renderPathForResource(annotatedInputs, k.yamlDirectory)))
"rendered %s", renderPathForResource(newInputs, k.yamlDirectory)))

return &pulumirpc.CreateResponse{
Id: fqObjName(annotatedInputs), Properties: inputsAndComputed,
Id: fqObjName(newInputs), Properties: inputsAndComputed,
}, nil
}

Expand All @@ -1817,7 +1811,7 @@ func (k *kubeProvider) Create(
Resources: resources,
ServerSideApply: k.serverSideApplyMode,
},
Inputs: annotatedInputs,
Inputs: newInputs,
Timeout: req.Timeout,
Preview: req.GetPreview(),
}
Expand Down Expand Up @@ -2065,8 +2059,10 @@ func (k *kubeProvider) Read(ctx context.Context, req *pulumirpc.ReadRequest) (*p
// initialize.
}

// Attempt to parse the inputs for this object. If parsing was unsuccessful, retain the old inputs.
liveInputs := parseLiveInputs(liveObj, oldInputs)
// Prune the live inputs to remove properties that are not present in the program inputs.
liveInputs := &unstructured.Unstructured{
Object: pruneMap(liveObj.Object, oldInputs.Object),
}

if readFromCluster {
// If no previous inputs were known, populate the inputs from the live cluster state for the resource.
Expand All @@ -2082,14 +2078,6 @@ func (k *kubeProvider) Read(ctx context.Context, req *pulumirpc.ReadRequest) (*p
}
}
}

// Cleanup some obviously non-input-ty fields.
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "creationTimestamp")
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "generation")
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "managedFields")
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "resourceVersion")
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "uid")
unstructured.RemoveNestedField(liveInputs.Object, "metadata", "annotations", lastAppliedConfigKey)
}

// TODO(lblackstone): not sure why this is needed
Expand Down Expand Up @@ -2240,15 +2228,7 @@ func (k *kubeProvider) Update(
return k.helmReleaseProvider.Update(ctx, req)
}
// Ignore old state; we'll get it from Kubernetes later.
oldInputs, _ := parseCheckpointObject(oldState)

annotatedInputs, err := k.withLastAppliedConfig(newInputs)
if err != nil {
return nil, pkgerrors.Wrapf(
err, "Failed to update resource %s/%s because of an error generating the %s value in "+
"`.metadata.annotations`",
newInputs.GetNamespace(), newInputs.GetName(), lastAppliedConfigKey)
}
oldInputs, liveInputs := parseCheckpointObject(oldState)

initialAPIVersion := initialAPIVersion(oldState, oldInputs)
fieldManagerOld := k.fieldManagerName(nil, oldState, oldInputs)
Expand All @@ -2258,14 +2238,14 @@ func (k *kubeProvider) Update(
if newResInputs.ContainsSecrets() {
_ = k.host.LogStatus(ctx, diag.Warning, urn, fmt.Sprintf(
"rendered file %s contains a secret value in plaintext",
renderPathForResource(annotatedInputs, k.yamlDirectory)))
renderPathForResource(newInputs, k.yamlDirectory)))
}
err := renderYaml(annotatedInputs, k.yamlDirectory)
err := renderYaml(newInputs, k.yamlDirectory)
if err != nil {
return nil, err
}

obj := checkpointObject(newInputs, annotatedInputs, newResInputs, initialAPIVersion, fieldManager)
obj := checkpointObject(newInputs, liveInputs, newResInputs, initialAPIVersion, fieldManager)
inputsAndComputed, err := plugin.MarshalProperties(
obj, plugin.MarshalOptions{
Label: fmt.Sprintf("%s.inputsAndComputed", label),
Expand All @@ -2278,7 +2258,7 @@ func (k *kubeProvider) Update(
}

_ = k.host.LogStatus(ctx, diag.Info, urn, fmt.Sprintf(
"rendered %s", renderPathForResource(annotatedInputs, k.yamlDirectory)))
"rendered %s", renderPathForResource(newInputs, k.yamlDirectory)))

return &pulumirpc.UpdateResponse{Properties: inputsAndComputed}, nil
}
Expand All @@ -2300,7 +2280,7 @@ func (k *kubeProvider) Update(
ServerSideApply: k.serverSideApplyMode,
},
Previous: oldInputs,
Inputs: annotatedInputs,
Inputs: newInputs,
Timeout: req.Timeout,
Preview: req.GetPreview(),
}
Expand Down Expand Up @@ -2637,7 +2617,10 @@ func (k *kubeProvider) serverSidePatch(oldInputs, newInputs *unstructured.Unstru
return nil, nil, err
}
} else {
liveInputs := parseLiveInputs(liveObject, oldInputs)
// Prune the live inputs to remove properties that are not present in the program inputs.
liveInputs := &unstructured.Unstructured{
Object: pruneMap(liveObject.Object, oldInputs.Object),
}

resources, err := k.getResources()
if err != nil {
Expand Down Expand Up @@ -2786,39 +2769,6 @@ func (k *kubeProvider) tryServerSidePatch(
return nil, nil, false, err
}

func (k *kubeProvider) withLastAppliedConfig(config *unstructured.Unstructured) (*unstructured.Unstructured, error) {
if k.serverSideApplyMode {
// Skip last-applied-config annotation if the resource supports server-side apply.
return config, nil
}

// CRDs are updated using a separate mechanism, so skip the last-applied-configuration annotation, and delete it
// if it was present from a previous update.
if clients.IsCRD(config) {
// Deep copy the config before returning.
config = config.DeepCopy()

annotations := getAnnotations(config)
delete(annotations, lastAppliedConfigKey)
config.SetAnnotations(annotations)
return config, nil
}

// Serialize the inputs and add the last-applied-configuration annotation.
marshaled, err := config.MarshalJSON()
if err != nil {
return nil, err
}

// Deep copy the config before returning.
config = config.DeepCopy()

annotations := getAnnotations(config)
annotations[lastAppliedConfigKey] = string(marshaled)
config.SetAnnotations(annotations)
return config, nil
}

// fieldManagerName returns the name to use for the Server-Side Apply fieldManager. The values are looked up with the
// following precedence:
// 1. Resource annotation (this will likely change to a typed option field in the next major release)
Expand Down Expand Up @@ -2934,6 +2884,20 @@ func getAnnotations(config *unstructured.Unstructured) map[string]string {
return annotations
}

// dropLastAppliedConfigAnnotation creates a copy of the Unstructured input minus the
// "kubectl.kubernetes.io/last-applied-configuration" annotation.
func dropLastAppliedConfigAnnotation(inputs *unstructured.Unstructured) *unstructured.Unstructured {
// Deep copy the inputs before returning.
inputs = inputs.DeepCopy()

annotations := inputs.GetAnnotations()
if annotations != nil {
delete(annotations, lastAppliedConfigKey)
inputs.SetAnnotations(annotations)
}
return inputs
}

// initialAPIVersion retrieves the initialAPIVersion property from the checkpoint file and falls back to using
// the version from the resource metadata if that property is not present.
func initialAPIVersion(state resource.PropertyMap, oldInputs *unstructured.Unstructured) string {
Expand Down Expand Up @@ -2973,21 +2937,6 @@ func checkpointObject(inputs, live *unstructured.Unstructured, fromInputs resour
}
}

// Ensure that the annotation we add for lastAppliedConfig is treated as a secret if any of the inputs were secret
// (the value of this annotation is a string-ified JSON so marking the entire thing as a secret is really the best
// that we can do).
if fromInputs.ContainsSecrets() || isSecretKind {
if _, has := object["metadata"]; has && object["metadata"].IsObject() {
metadata := object["metadata"].ObjectValue()
if _, has := metadata["annotations"]; has && metadata["annotations"].IsObject() {
annotations := metadata["annotations"].ObjectValue()
if lastAppliedConfig, has := annotations[lastAppliedConfigKey]; has && !lastAppliedConfig.IsSecret() {
annotations[lastAppliedConfigKey] = resource.MakeSecret(lastAppliedConfig)
}
}
}
}

object["__inputs"] = resource.NewObjectProperty(inputsPM)
object[initialAPIVersionKey] = resource.NewStringProperty(initialAPIVersion)
object[fieldManagerKey] = resource.NewStringProperty(fieldManager)
Expand Down Expand Up @@ -3055,54 +3004,6 @@ func canonicalNamespace(ns string) string {
// deleteResponse causes the resource to be deleted from the state.
var deleteResponse = &pulumirpc.ReadResponse{Id: "", Properties: nil}

// parseLastAppliedConfig attempts to find and parse an annotation that records the last applied configuration for the
// given live object state.
func parseLastAppliedConfig(live *unstructured.Unstructured) *unstructured.Unstructured {
// If `kubectl.kubernetes.io/last-applied-configuration` metadata annotation is present, parse it into a real object
// and use it as the current set of live inputs. Otherwise, return nil.
if live == nil {
return nil
}

annotations := live.GetAnnotations()
if annotations == nil {
return nil
}
lastAppliedConfig, ok := annotations[lastAppliedConfigKey]
if !ok {
return nil
}

liveInputs := &unstructured.Unstructured{}
if err := liveInputs.UnmarshalJSON([]byte(lastAppliedConfig)); err != nil {
return nil
}
return liveInputs
}

// parseLiveInputs attempts to parse the provider inputs that produced the given live object out of the object's state.
// This is used by Read.
func parseLiveInputs(live, oldInputs *unstructured.Unstructured) *unstructured.Unstructured {
// First try to find and parse a `kubectl.kubernetes.io/last-applied-configuration` metadata anotation. If that
// succeeds, we are done.
if inputs := parseLastAppliedConfig(live); inputs != nil {
return inputs
}

// If no such annotation was present--or if parsing failed--either retain the old inputs if they exist, or
// attempt to propagate the live object's GVK, any Pulumi-generated autoname and its annotation, and return
// the result.
if oldInputs != nil && len(oldInputs.Object) > 0 {
return oldInputs
}

inputs := &unstructured.Unstructured{Object: map[string]interface{}{}}
inputs.SetGroupVersionKind(live.GroupVersionKind())
metadata.AdoptOldAutonameIfUnnamed(inputs, live)

return inputs
}

// convertPatchToDiff converts the given JSON merge patch to a Pulumi detailed diff.
func convertPatchToDiff(
patch, oldLiveState, newInputs, oldInputs map[string]interface{}, forceNewFields ...string,
Expand Down

0 comments on commit d91155e

Please sign in to comment.