-
Notifications
You must be signed in to change notification settings - Fork 106
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add generic crd upgrade safety preflight check
and some initial validations for handling scope changes and removal of existing stored versions Signed-off-by: everettraven <everettraven@gmail.com>
- Loading branch information
1 parent
f249ba3
commit e12d91d
Showing
37 changed files
with
17,140 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| // Copyright 2024 The Carvel Authors. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package crdupgradesafety | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core" | ||
| ctldgraph "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/diffgraph" | ||
| "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/preflight" | ||
| v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" | ||
| apierrors "k8s.io/apimachinery/pkg/api/errors" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| ) | ||
|
|
||
| var _ preflight.Check = (*Preflight)(nil) | ||
|
|
||
| // Preflight is an implementation of preflight.Check | ||
| // to make it easier to add crd upgrade validation | ||
| // as a preflight check | ||
| type Preflight struct { | ||
| depsFactory cmdcore.DepsFactory | ||
| enabled bool | ||
| validator *Validator | ||
| } | ||
|
|
||
| func NewPreflight(df cmdcore.DepsFactory, enabled bool) *Preflight { | ||
| return &Preflight{ | ||
| depsFactory: df, | ||
| enabled: enabled, | ||
| validator: &Validator{ | ||
| Validations: []Validation{ | ||
| NewValidationFunc("NoScopeChange", NoScopeChange), | ||
| NewValidationFunc("NoStoredVersionRemoved", NoStoredVersionRemoved), | ||
| }, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| func (p *Preflight) Enabled() bool { | ||
| return p.enabled | ||
| } | ||
|
|
||
| func (p *Preflight) SetEnabled(enabled bool) { | ||
| p.enabled = enabled | ||
| } | ||
|
|
||
| func (p *Preflight) SetConfig(_ preflight.CheckConfig) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (p *Preflight) Run(ctx context.Context, changeGraph *ctldgraph.ChangeGraph) error { | ||
| dCli, err := p.depsFactory.DynamicClient(cmdcore.DynamicClientOpts{}) | ||
| if err != nil { | ||
| return fmt.Errorf("getting dynamic client: %w", err) | ||
| } | ||
| crdCli := dCli.Resource(v1.SchemeGroupVersion.WithResource("customresourcedefinitions")) | ||
|
|
||
| validateErrs := []error{} | ||
| for _, change := range changeGraph.All() { | ||
| // Loop through all the changes looking for "upsert" operations on | ||
| // a CRD. "upsert" is used for create + update operations | ||
| if change.Change.Op() != ctldgraph.ActualChangeOpUpsert { | ||
| continue | ||
| } | ||
| res := change.Change.Resource() | ||
| if res.GroupVersion().WithKind(res.Kind()) != v1.SchemeGroupVersion.WithKind("CustomResourceDefinition") { | ||
| continue | ||
| } | ||
|
|
||
| // to properly determine if this is an update operation, attempt to fetch | ||
| // the "old" CRD from the cluster | ||
| uOldCRD, err := crdCli.Get(ctx, res.Name(), metav1.GetOptions{}) | ||
| if err != nil { | ||
| // if the resource is not found, this "upsert" operation | ||
| // translates to a "create" request being made. Skip this change | ||
| if apierrors.IsNotFound(err) { | ||
| continue | ||
| } | ||
|
|
||
| return fmt.Errorf("checking for existing CRD resource: %w", err) | ||
| } | ||
|
|
||
| oldCRD := &v1.CustomResourceDefinition{} | ||
| s := runtime.NewScheme() | ||
| if err := v1.AddToScheme(s); err != nil { | ||
| return fmt.Errorf("adding apiextension apis to scheme: %w", err) | ||
| } | ||
| if err := s.Convert(uOldCRD, oldCRD, nil); err != nil { | ||
| return fmt.Errorf("couldn't convert old CRD resource to a CRD object: %w", err) | ||
| } | ||
|
|
||
| newCRD := &v1.CustomResourceDefinition{} | ||
| if err := res.AsUncheckedTypedObj(newCRD); err != nil { | ||
| return fmt.Errorf("couldn't convert new CRD resource to a CRD object: %w", err) | ||
| } | ||
|
|
||
| if err = p.validator.Validate(*oldCRD, *newCRD); err != nil { | ||
| validateErrs = append(validateErrs, err) | ||
| } | ||
| } | ||
|
|
||
| if len(validateErrs) > 0 { | ||
| baseErr := errors.New("validation for safe CRD upgrades failed") | ||
| return errors.Join(append([]error{baseErr}, validateErrs...)...) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // Copyright 2024 The Carvel Authors. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package crdupgradesafety | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" | ||
| "k8s.io/apimachinery/pkg/util/sets" | ||
| ) | ||
|
|
||
| // Validation is a representation of a validation to run | ||
| // against a CRD being upgraded | ||
| type Validation interface { | ||
| // Validate contains the actual validation logic. An error being | ||
| // returned means validation has failed | ||
| Validate(old, new v1.CustomResourceDefinition) error | ||
| // Name returns a human-readable name for the validation | ||
| Name() string | ||
| } | ||
|
|
||
| // ValidateFunc is a function to validate a CustomResourceDefinition | ||
| // for safe upgrades. It accepts the old and new CRDs and returns an | ||
| // error if performing an upgrade from old -> new is unsafe. | ||
| type ValidateFunc func(old, new v1.CustomResourceDefinition) error | ||
|
|
||
| // ValidationFunc is a helper to wrap a ValidateFunc | ||
| // as an implementation of the Validation interface | ||
| type ValidationFunc struct { | ||
| name string | ||
| validateFunc ValidateFunc | ||
| } | ||
|
|
||
| func NewValidationFunc(name string, vfunc ValidateFunc) Validation { | ||
| return &ValidationFunc{ | ||
| name: name, | ||
| validateFunc: vfunc, | ||
| } | ||
| } | ||
|
|
||
| func (vf *ValidationFunc) Name() string { | ||
| return vf.name | ||
| } | ||
|
|
||
| func (vf *ValidationFunc) Validate(old, new v1.CustomResourceDefinition) error { | ||
| return vf.validateFunc(old, new) | ||
| } | ||
|
|
||
| type Validator struct { | ||
| Validations []Validation | ||
| } | ||
|
|
||
| func (v *Validator) Validate(old, new v1.CustomResourceDefinition) error { | ||
| validateErrs := []error{} | ||
| for _, validation := range v.Validations { | ||
| if err := validation.Validate(old, new); err != nil { | ||
| formattedErr := fmt.Errorf("CustomResourceDefinition %s failed upgrade safety validation. %q validation failed: %w", | ||
| new.Name, validation.Name(), err) | ||
|
|
||
| validateErrs = append(validateErrs, formattedErr) | ||
| } | ||
| } | ||
| if len(validateErrs) > 0 { | ||
| return errors.Join(validateErrs...) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func NoScopeChange(old, new v1.CustomResourceDefinition) error { | ||
| if old.Spec.Scope != new.Spec.Scope { | ||
| return fmt.Errorf("scope changed from %q to %q", old.Spec.Scope, new.Spec.Scope) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func NoStoredVersionRemoved(old, new v1.CustomResourceDefinition) error { | ||
| newVersions := sets.New[string]() | ||
| for _, version := range new.Spec.Versions { | ||
| if !newVersions.Has(version.Name) { | ||
| newVersions.Insert(version.Name) | ||
| } | ||
| } | ||
|
|
||
| for _, storedVersion := range old.Status.StoredVersions { | ||
| if !newVersions.Has(storedVersion) { | ||
| return fmt.Errorf("stored version %q removed", storedVersion) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.