Skip to content
This repository has been archived by the owner on Apr 25, 2023. It is now read-only.

Commit

Permalink
Merge pull request #1355 from RolandMa1986/feat/kubefed-deleteoption
Browse files Browse the repository at this point in the history
Feature: support DeleteOptions when deleting resources in member clusters
  • Loading branch information
k8s-ci-robot committed Feb 18, 2021
2 parents 6ac5df1 + b7061be commit 485bcf0
Show file tree
Hide file tree
Showing 8 changed files with 303 additions and 11 deletions.
6 changes: 3 additions & 3 deletions pkg/client/generic/genericclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type Client interface {
Create(ctx context.Context, obj runtime.Object) error
Get(ctx context.Context, obj runtime.Object, namespace, name string) error
Update(ctx context.Context, obj runtime.Object) error
Delete(ctx context.Context, obj runtime.Object, namespace, name string) error
Delete(ctx context.Context, obj runtime.Object, namespace, name string, opts ...client.DeleteOption) error
List(ctx context.Context, obj runtime.Object, namespace string, opts ...client.ListOption) error
UpdateStatus(ctx context.Context, obj runtime.Object) error
}
Expand Down Expand Up @@ -71,14 +71,14 @@ func (c *genericClient) Update(ctx context.Context, obj runtime.Object) error {
return c.client.Update(ctx, obj)
}

func (c *genericClient) Delete(ctx context.Context, obj runtime.Object, namespace, name string) error {
func (c *genericClient) Delete(ctx context.Context, obj runtime.Object, namespace, name string, opts ...client.DeleteOption) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return err
}
accessor.SetNamespace(namespace)
accessor.SetName(name)
return c.client.Delete(ctx, obj)
return c.client.Delete(ctx, obj, opts...)
}

func (c *genericClient) List(ctx context.Context, obj runtime.Object, namespace string, opts ...client.ListOption) error {
Expand Down
16 changes: 13 additions & 3 deletions pkg/controller/sync/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"

"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/kubefed/pkg/apis/core/typeconfig"
fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
Expand Down Expand Up @@ -481,8 +483,16 @@ func (s *KubeFedSyncController) ensureDeletion(fedResource FederatedResource) ut
return util.StatusAllOK
}

klog.V(2).Infof("Deserializing delete options of %s %q", kind, key)
opts, err := util.GetDeleteOptions(obj)
if err != nil {
wrappedErr := errors.Wrapf(err, "failed to deserialize delete options of %s %q", kind, key)
runtime.HandleError(wrappedErr)
return util.StatusError
}

klog.V(2).Infof("Deleting resources managed by %s %q from member clusters.", kind, key)
recheckRequired, err := s.deleteFromClusters(fedResource)
recheckRequired, err := s.deleteFromClusters(fedResource, opts...)
if err != nil {
wrappedErr := errors.Wrapf(err, "failed to delete %s %q", kind, key)
runtime.HandleError(wrappedErr)
Expand Down Expand Up @@ -513,7 +523,7 @@ func (s *KubeFedSyncController) removeManagedLabel(gvk schema.GroupVersionKind,
return nil
}

func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource) (bool, error) {
func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource, opts ...client.DeleteOption) (bool, error) {
gvk := fedResource.TargetGVK()
qualifiedName := fedResource.TargetName()

Expand Down Expand Up @@ -542,7 +552,7 @@ func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource
// namespace is no longer cached.
dispatcher.RemoveManagedLabel(clusterName, clusterObj)
} else {
dispatcher.Delete(clusterName)
dispatcher.Delete(clusterName, opts...)
}
})
if err != nil {
Expand Down
6 changes: 4 additions & 2 deletions pkg/controller/sync/dispatch/managed.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"

"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/kubefed/pkg/client/generic"
"sigs.k8s.io/kubefed/pkg/controller/sync/status"
"sigs.k8s.io/kubefed/pkg/controller/util"
Expand Down Expand Up @@ -245,10 +247,10 @@ func (d *managedDispatcherImpl) Update(clusterName string, clusterObj *unstructu
})
}

func (d *managedDispatcherImpl) Delete(clusterName string) {
func (d *managedDispatcherImpl) Delete(clusterName string, opts ...client.DeleteOption) {
d.RecordStatus(clusterName, status.DeletionTimedOut, nil)

d.unmanagedDispatcher.Delete(clusterName)
d.unmanagedDispatcher.Delete(clusterName, opts...)
}

func (d *managedDispatcherImpl) RemoveManagedLabel(clusterName string, clusterObj *unstructured.Unstructured) {
Expand Down
8 changes: 5 additions & 3 deletions pkg/controller/sync/dispatch/unmanaged.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/klog/v2"

"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/kubefed/pkg/client/generic"
"sigs.k8s.io/kubefed/pkg/controller/sync/status"
"sigs.k8s.io/kubefed/pkg/controller/util"
Expand All @@ -41,7 +43,7 @@ const eventTemplate = "%s %s %q in cluster %q"
type UnmanagedDispatcher interface {
OperationDispatcher

Delete(clusterName string)
Delete(clusterName string, opts ...client.DeleteOption)
RemoveManagedLabel(clusterName string, clusterObj *unstructured.Unstructured)
}

Expand Down Expand Up @@ -72,7 +74,7 @@ func (d *unmanagedDispatcherImpl) Wait() (bool, error) {
return d.dispatcher.Wait()
}

func (d *unmanagedDispatcherImpl) Delete(clusterName string) {
func (d *unmanagedDispatcherImpl) Delete(clusterName string, opts ...client.DeleteOption) {
start := time.Now()
d.dispatcher.incrementOperationsInitiated()
const op = "delete"
Expand All @@ -87,7 +89,7 @@ func (d *unmanagedDispatcherImpl) Delete(clusterName string) {

obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(d.targetGVK)
err := client.Delete(context.Background(), obj, targetName.Namespace, targetName.Name)
err := client.Delete(context.Background(), obj, targetName.Namespace, targetName.Name, opts...)
if apierrors.IsNotFound(err) {
err = nil
}
Expand Down
73 changes: 73 additions & 0 deletions pkg/controller/util/deletionannotation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
"encoding/json"

"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
// DeleteOptionAnnotation contains options for delete
// while deleting resources for member clusters.
DeleteOptionAnnotation = "kubefed.io/deleteoption"
)

// GetDeleteOptions return delete options from the annotation
func GetDeleteOptions(obj *unstructured.Unstructured) ([]client.DeleteOption, error) {
options := make([]client.DeleteOption, 0)
annotations := obj.GetAnnotations()
if annotations == nil {
return options, nil
}

if optStr, ok := annotations[DeleteOptionAnnotation]; ok {
opt := &metav1.DeleteOptions{}
if err := json.Unmarshal([]byte(optStr), opt); err != nil {
return nil, errors.Wrapf(err, "could not deserialize delete options from annotation value '%s'", optStr)
}
clientOpt := &client.DeleteOptions{}
clientOpt.GracePeriodSeconds = opt.GracePeriodSeconds
clientOpt.PropagationPolicy = opt.PropagationPolicy
clientOpt.Preconditions = opt.Preconditions
options = append(options, clientOpt)
}
return options, nil
}

// ApplyDeleteOptions set the DeleteOptions on the annotation
func ApplyDeleteOptions(obj *unstructured.Unstructured, opts ...client.DeleteOption) error {
opt := client.DeleteOptions{}
opt.ApplyOptions(opts)
deleteOpts := opt.AsDeleteOptions()
optBytes, err := json.Marshal(deleteOpts)
if err != nil {
return errors.Wrapf(err, "could not serialize delete options from object '%v'", deleteOpts)
}

annotations := obj.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations[DeleteOptionAnnotation] = string(optBytes)
obj.SetAnnotations(annotations)
return nil
}
86 changes: 86 additions & 0 deletions pkg/controller/util/deletionannotation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func TestDeleteOptions(t *testing.T) {
fedObj := &unstructured.Unstructured{}

fedObjOrphan := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"annotations": map[string]interface{}{
DeleteOptionAnnotation: "{\"propagationPolicy\":\"Orphan\"}",
},
},
},
}

fedObjGrace := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"annotations": map[string]interface{}{
DeleteOptionAnnotation: "{\"gracePeriodSeconds\":5}",
},
},
},
}

fedObjGraceAndOrphan := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"annotations": map[string]interface{}{
DeleteOptionAnnotation: "{\"propagationPolicy\":\"Orphan\",\"gracePeriodSeconds\":5}",
},
},
},
}

actOpt0 := client.DeleteOptions{}
opt0, _ := GetDeleteOptions(fedObj)
actOpt0.ApplyOptions(opt0)
expOpt0 := client.DeleteOptions{}
assert.Equal(t, expOpt0.AsDeleteOptions(), actOpt0.AsDeleteOptions())

actOpt1 := client.DeleteOptions{}
opt1, _ := GetDeleteOptions(fedObjOrphan)
actOpt1.ApplyOptions(opt1)
prop := metav1.DeletePropagationOrphan
expOpt1 := client.DeleteOptions{PropagationPolicy: &prop}
assert.Equal(t, expOpt1.AsDeleteOptions(), actOpt1.AsDeleteOptions())

actOpt2 := client.DeleteOptions{}
opt2, _ := GetDeleteOptions(fedObjGrace)
actOpt2.ApplyOptions(opt2)
seconds := int64(5)
expOpt2 := client.DeleteOptions{GracePeriodSeconds: &seconds}
assert.Equal(t, expOpt2.AsDeleteOptions(), actOpt2.AsDeleteOptions())

actOpt3 := client.DeleteOptions{}
opt3, _ := GetDeleteOptions(fedObjGraceAndOrphan)
actOpt3.ApplyOptions(opt3)
expOpt3 := client.DeleteOptions{GracePeriodSeconds: &seconds, PropagationPolicy: &prop}
assert.Equal(t, expOpt3.AsDeleteOptions(), actOpt3.AsDeleteOptions())
}
63 changes: 63 additions & 0 deletions test/common/crudtester.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
jsonpatch "github.com/evanphx/json-patch"
"github.com/pkg/errors"

appsv1 "k8s.io/api/apps/v1"
apiv1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -35,6 +36,8 @@ import (
kubeclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/kubefed/pkg/apis/core/common"
"sigs.k8s.io/kubefed/pkg/apis/core/typeconfig"
fedv1a1 "sigs.k8s.io/kubefed/pkg/apis/core/v1alpha1"
Expand Down Expand Up @@ -362,6 +365,66 @@ func (c *FederatedTypeCrudTester) CheckDelete(fedObject *unstructured.Unstructur
}
}

func (c *FederatedTypeCrudTester) SetDeleteOption(fedObject *unstructured.Unstructured, opts ...client.DeleteOption) {
apiResource := c.typeConfig.GetFederatedType()
qualifiedName := util.NewQualifiedName(fedObject)
kind := apiResource.Kind
_, err := c.updateObject(apiResource, fedObject, func(obj *unstructured.Unstructured) {
err := util.ApplyDeleteOptions(obj, opts...)
if err != nil {
c.tl.Fatalf("Error apply delete options for %s %q: %v", kind, qualifiedName, err)
}
})
if err != nil {
c.tl.Fatalf("Error updating %s %q: %v", kind, qualifiedName, err)
}
}

func (c *FederatedTypeCrudTester) CheckReplicaSet(fedObject *unstructured.Unstructured) {
lb, ok, _ := unstructured.NestedStringMap(fedObject.Object, "spec", "selector", "matchLabels")
if !ok {
c.tl.Fatal("Failed to get matchLabels on the target deployment")
}

matchingLabels := (client.MatchingLabels)(lb)

for clusterName := range c.testClusters {
clusterConfig := c.testClusters[clusterName].Config

kubeClient := kubeclientset.NewForConfigOrDie(clusterConfig)
WaitForNamespaceOrDie(c.tl, kubeClient, clusterName, fedObject.GetNamespace(),
c.waitInterval, 30*time.Second)

clusterClient := genericclient.NewForConfigOrDie(clusterConfig)

c.tl.Log("Checking that the ReplicaSet still exists in every cluster")

err := wait.PollImmediate(c.waitInterval, wait.ForeverTestTimeout, func() (bool, error) {
objList := &appsv1.ReplicaSetList{}
err := clusterClient.List(context.TODO(), objList, fedObject.GetNamespace(), matchingLabels)
if err != nil {
return false, errors.Errorf("Error retrieving ReplicatSet: %v", err)
}

if len(objList.Items) == 0 {
return false, errors.Errorf("ReplicatSet was unexpectedly deleted from cluster %q", clusterName)
}

c.tl.Log("Checking that OwnerReferences has been removed from the ReplicaSet")
hasOwner := false
for _, rs := range objList.Items {
if len(rs.OwnerReferences) > 0 {
hasOwner = true
}
}
return !hasOwner, nil
})
if err != nil {
c.tl.Fatalf("Failed to confirm whether ReplicatSet is in cluster %q: %v", clusterName, err)
}
}
}

// CheckPropagation checks propagation for the crud tester's clients
func (c *FederatedTypeCrudTester) CheckPropagation(fedObject *unstructured.Unstructured) {
federatedKind := c.typeConfig.GetFederatedType().Kind
Expand Down
Loading

0 comments on commit 485bcf0

Please sign in to comment.