Skip to content

Commit

Permalink
pkg/action: various operator uninstall and catalog improvements (#15)
Browse files Browse the repository at this point in the history
- new helper to get object key from object
- new helper to re-use code that waits for object deletion
- operator uninstall: add DeleteOperatorGroupNames option that is used
  to filter which operator groups are deleted.
- operator uninstall: always wait for objects to be deleted.
  • Loading branch information
joelanford committed Aug 10, 2020
1 parent 5ac9c8d commit 25407c3
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 73 deletions.
11 changes: 5 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwo
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsouza/fake-gcs-server v1.7.0/go.mod h1:5XIRs4YvwNbNoz+1JF8j6KLAyDh7RHGAyAK3EP2EsNk=
github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
Expand Down Expand Up @@ -375,6 +376,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
Expand Down Expand Up @@ -478,6 +480,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
Expand All @@ -490,6 +493,7 @@ github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand Down Expand Up @@ -957,8 +961,6 @@ k8s.io/api v0.16.7/go.mod h1:oUAiGRgo4t+5yqcxjOu5LoHT3wJ8JSbgczkaFYS5L7I=
k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4=
k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8=
k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
k8s.io/api v0.18.4 h1:8x49nBRxuXGUlDlwlWd3RMY1SayZrzFfxea3UZSkFw4=
k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4=
k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE=
k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=
k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY=
Expand All @@ -975,8 +977,6 @@ k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZ
k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
k8s.io/apimachinery v0.18.4 h1:ST2beySjhqwJoIFk6p7Hp5v5O0hYY6Gngq/gUYXTPIA=
k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag=
k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
Expand All @@ -993,8 +993,6 @@ k8s.io/client-go v0.16.7/go.mod h1:9kEMEeuy2LdsHHXoU2Skqh+SDso+Yhkxd/0tltvswDE=
k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI=
k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8=
k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
k8s.io/client-go v0.18.4 h1:un55V1Q/B3JO3A76eS0kUSywgGK/WR3BQ8fHQjNa6Zc=
k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g=
k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw=
k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE=
Expand All @@ -1017,6 +1015,7 @@ k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/kube-aggregator v0.18.0/go.mod h1:ateewQ5QbjMZF/dihEFXwaEwoA4v/mayRvzfmvb6eqI=
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
Expand Down
17 changes: 3 additions & 14 deletions internal/pkg/action/catalog_add.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

"github.com/operator-framework/kubectl-operator/internal/pkg/catalog"
Expand Down Expand Up @@ -189,11 +188,7 @@ func (a *CatalogAdd) createRegistryPod(ctx context.Context, cs *v1alpha1.Catalog
return nil, fmt.Errorf("create registry pod: %v", err)
}

podKey, err := client.ObjectKeyFromObject(pod)
if err != nil {
return nil, fmt.Errorf("get registry pod key: %v", err)
}

podKey := objectKeyForObject(pod)
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err := a.config.Client.Get(ctx, podKey, pod); err != nil {
return fmt.Errorf("get registry pod: %v", err)
Expand Down Expand Up @@ -235,10 +230,7 @@ func (a *CatalogAdd) updateCatalogSource(ctx context.Context, cs *v1alpha1.Catal
"operators.operatorframework.io/inject-bundle-mode": a.InjectBundleMode,
"operators.operatorframework.io/injected-bundles": string(injectedBundlesJSON),
}
csKey, err := client.ObjectKeyFromObject(cs)
if err != nil {
return fmt.Errorf("get catalogsource key: %v", err)
}
csKey := objectKeyForObject(cs)
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err := a.config.Client.Get(ctx, csKey, cs); err != nil {
return fmt.Errorf("get catalog source: %v", err)
Expand All @@ -254,10 +246,7 @@ func (a *CatalogAdd) updateCatalogSource(ctx context.Context, cs *v1alpha1.Catal
}

func (a *CatalogAdd) waitForCatalogSourceReady(ctx context.Context, cs *v1alpha1.CatalogSource) error {
csKey, err := client.ObjectKeyFromObject(cs)
if err != nil {
return fmt.Errorf("get catalogsource key: %v", err)
}
csKey := objectKeyForObject(cs)
if err := wait.PollImmediateUntil(time.Millisecond*250, func() (bool, error) {
if err := a.config.Client.Get(ctx, csKey, cs); err != nil {
return false, err
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/action/catalog_remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ func (r *CatalogRemove) Run(ctx context.Context) error {
if err := r.config.Client.Delete(ctx, &cs); err != nil {
return fmt.Errorf("delete catalogsource %q: %v", cs.Name, err)
}
return nil
return waitForDeletion(ctx, r.config.Client, &cs)
}
40 changes: 40 additions & 0 deletions internal/pkg/action/helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package action

import (
"context"
"fmt"
"strings"
"time"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

func objectKeyForObject(obj controllerutil.Object) types.NamespacedName {
return types.NamespacedName{
Namespace: obj.GetNamespace(),
Name: obj.GetName(),
}
}

func waitForDeletion(ctx context.Context, cl client.Client, objs ...controllerutil.Object) error {
for _, obj := range objs {
obj := obj
lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
key := objectKeyForObject(obj)
if err := wait.PollImmediateUntil(250*time.Millisecond, func() (bool, error) {
if err := cl.Get(ctx, key, obj); apierrors.IsNotFound(err) {
return true, nil
} else if err != nil {
return false, err
}
return false, nil
}, ctx.Done()); err != nil {
return fmt.Errorf("wait for %s %q deleted: %v", lowerKind, key.Name, err)
}
}
return nil
}
98 changes: 46 additions & 52 deletions internal/pkg/action/operator_uninstall.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"fmt"
"strings"
"time"

v1 "github.com/operator-framework/api/pkg/operators/v1"
"github.com/operator-framework/api/pkg/operators/v1alpha1"
Expand All @@ -13,7 +12,6 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/yaml"
Expand All @@ -22,10 +20,11 @@ import (
type OperatorUninstall struct {
config *Configuration

Package string
DeleteOperatorGroup bool
DeleteCRDs bool
DeleteAll bool
Package string
DeleteAll bool
DeleteCRDs bool
DeleteOperatorGroups bool
DeleteOperatorGroupNames []string

Logf func(string, ...interface{})
}
Expand All @@ -38,15 +37,16 @@ func NewOperatorUninstall(cfg *Configuration) *OperatorUninstall {
}

func (u *OperatorUninstall) BindFlags(fs *pflag.FlagSet) {
fs.BoolVar(&u.DeleteOperatorGroup, "delete-operator-group", false, "delete operator group if no other operators remain")
fs.BoolVarP(&u.DeleteAll, "delete-all", "X", false, "enable all delete flags")
fs.BoolVar(&u.DeleteCRDs, "delete-crds", false, "delete all owned CRDs and all CRs")
fs.BoolVarP(&u.DeleteAll, "delete-add", "X", false, "enable all delete flags")
fs.BoolVar(&u.DeleteOperatorGroups, "delete-operator-groups", false, "delete operator group if no other operators remain")
fs.StringSliceVar(&u.DeleteOperatorGroupNames, "delete-operator-group-names", nil, "delete operator group if no other operators remain")
}

func (u *OperatorUninstall) Run(ctx context.Context) error {
if u.DeleteAll {
u.DeleteCRDs = true
u.DeleteOperatorGroup = true
u.DeleteOperatorGroups = true
}

subs := v1alpha1.SubscriptionList{}
Expand All @@ -66,6 +66,9 @@ func (u *OperatorUninstall) Run(ctx context.Context) error {
return fmt.Errorf("operator package %q not found", u.Package)
}

// Since the install plan is owned by the subscription, we need to
// read all of the resource references from the install plan before
// deleting the subscription.
var crds, csvs, others []controllerutil.Object
if sub.Status.InstallPlanRef != nil {
ipKey := types.NamespacedName{
Expand All @@ -79,56 +82,52 @@ func (u *OperatorUninstall) Run(ctx context.Context) error {
}
}

if err := u.config.Client.Delete(ctx, sub); err != nil {
return fmt.Errorf("delete subscription %q: %v", sub.Name, err)
// Delete the subscription first, so that no further installs or upgrades
// of the operator occur while we're cleaning up.
if err := u.deleteObjects(ctx, sub); err != nil {
return err
}
u.Logf("subscription %q deleted", sub.Name)

if u.DeleteCRDs {
if err := u.deleteCRDs(ctx, crds); err != nil {
// Ensure CustomResourceDefinitions are deleted next, so that the operator
// has a chance to handle CRs that have finalizers.
if err := u.deleteObjects(ctx, crds...); err != nil {
return err
}
}

if err := u.deleteObjects(ctx, false, csvs); err != nil {
return err
}

if err := u.deleteObjects(ctx, false, others); err != nil {
// Delete CSVs and all other objects created by the install plan.
objects := append(csvs, others...)
if err := u.deleteObjects(ctx, objects...); err != nil {
return err
}

if u.DeleteOperatorGroup {
if u.DeleteOperatorGroups {
subs := v1alpha1.SubscriptionList{}
if err := u.config.Client.List(ctx, &subs, client.InNamespace(u.config.Namespace)); err != nil {
return fmt.Errorf("list clusterserviceversions: %v", err)
return fmt.Errorf("list subscriptions: %v", err)
}
// If there are no subscriptions left, delete the operator group(s).
if len(subs.Items) == 0 {
ogs := v1.OperatorGroupList{}
if err := u.config.Client.List(ctx, &ogs, client.InNamespace(u.config.Namespace)); err != nil {
return fmt.Errorf("list operatorgroups: %v", err)
}
for _, og := range ogs.Items {
og := og
if err := u.config.Client.Delete(ctx, &og); err != nil {
return fmt.Errorf("delete operatorgroup %q: %v", og.Name, err)
if len(u.DeleteOperatorGroupNames) == 0 || contains(u.DeleteOperatorGroupNames, og.GetName()) {
if err := u.deleteObjects(ctx, &og); err != nil {
return err
}
}
u.Logf("operatorgroup %q deleted", og.Name)
}
}
}

return nil
}

func (u *OperatorUninstall) deleteCRDs(ctx context.Context, crds []controllerutil.Object) error {
if err := u.deleteObjects(ctx, true, crds); err != nil {
return err
}
return nil
}

func (u *OperatorUninstall) deleteObjects(ctx context.Context, waitForDelete bool, objs []controllerutil.Object) error {
func (u *OperatorUninstall) deleteObjects(ctx context.Context, objs ...controllerutil.Object) error {
for _, obj := range objs {
obj := obj
lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
Expand All @@ -137,24 +136,8 @@ func (u *OperatorUninstall) deleteObjects(ctx context.Context, waitForDelete boo
} else if err == nil {
u.Logf("%s %q deleted", lowerKind, obj.GetName())
}
if waitForDelete {
key, err := client.ObjectKeyFromObject(obj)
if err != nil {
return fmt.Errorf("get %s key: %v", lowerKind, err)
}
if err := wait.PollImmediateUntil(250*time.Millisecond, func() (bool, error) {
if err := u.config.Client.Get(ctx, key, obj); apierrors.IsNotFound(err) {
return true, nil
} else if err != nil {
return false, err
}
return false, nil
}, ctx.Done()); err != nil {
return fmt.Errorf("wait for %s deleted: %v", lowerKind, err)
}
}
}
return nil
return waitForDeletion(ctx, u.config.Client, objs...)
}

func (u *OperatorUninstall) getInstallPlanResources(ctx context.Context, installPlanKey types.NamespacedName) (crds, csvs, others []controllerutil.Object, err error) {
Expand All @@ -164,11 +147,8 @@ func (u *OperatorUninstall) getInstallPlanResources(ctx context.Context, install
}

for _, step := range installPlan.Status.Plan {
if step.Status != v1alpha1.StepStatusCreated {
continue
}
obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
lowerKind := strings.ToLower(step.Resource.Kind)
obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
if err := yaml.Unmarshal([]byte(step.Resource.Manifest), &obj.Object); err != nil {
return nil, nil, nil, fmt.Errorf("parse %s manifest %q: %v", lowerKind, step.Resource.Name, err)
}
Expand All @@ -191,8 +171,22 @@ func (u *OperatorUninstall) getInstallPlanResources(ctx context.Context, install
case csvKind:
csvs = append(csvs, obj)
default:
// Skip non-CRD/non-CSV resources in the install plan that were not created by the install plan.
// This means we avoid deleting things like the default service account.
if step.Status != v1alpha1.StepStatusCreated {
continue
}
others = append(others, obj)
}
}
return crds, csvs, others, nil
}

func contains(haystack []string, needle string) bool {
for _, n := range haystack {
if n == needle {
return true
}
}
return false
}

0 comments on commit 25407c3

Please sign in to comment.