Skip to content

Commit

Permalink
Add DeletionConfirmation admission controller
Browse files Browse the repository at this point in the history
The current way of deletion of Shoot resources is a left over from when the
Gardener was working as TPR operator and external validation webhooks were not
available.

DeletionConfirmation admission controller fixes this issue by allowing deletion
of resources to happen when a Shoot resource is annotated with
`confirmation.garden.sapcloud.io/deletion=true`

It is disabled by default to preserve the old behavior.
  • Loading branch information
Martin Vladev committed May 31, 2018
1 parent 54ec6ba commit cce0385
Show file tree
Hide file tree
Showing 7 changed files with 329 additions and 11 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ start-api:
@go run cmd/gardener-apiserver/main.go \
--authentication-kubeconfig ~/.kube/config \
--authorization-kubeconfig ~/.kube/config \
--enable-admission-plugins=ResourceReferenceManager,ShootSeedManager,ShootDNSHostedZone,ShootValidator,ShootQuotaValidator \
--enable-admission-plugins=DeletionConfirmation \
--etcd-servers=http://$(shell minikube ip):32379 \
--kubeconfig ~/.kube/config \
--tls-cert-file ~/.minikube/apiserver.crt \
Expand Down
16 changes: 6 additions & 10 deletions cmd/gardener-apiserver/app/gardener_apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer"
gardenclientset "github.com/gardener/gardener/pkg/client/garden/clientset/internalversion"
gardeninformers "github.com/gardener/gardener/pkg/client/garden/informers/internalversion"
deletionconfirmation "github.com/gardener/gardener/plugin/pkg/global/deletionconfirmation"
resourcereferencemanager "github.com/gardener/gardener/plugin/pkg/global/resourcereferencemanager"
shootdnshostedzone "github.com/gardener/gardener/plugin/pkg/shoot/dnshostedzone"
shootquotavalidator "github.com/gardener/gardener/plugin/pkg/shoot/quotavalidator"
Expand Down Expand Up @@ -116,9 +117,11 @@ func (o *Options) complete() error {
shootseedmanager.Register(o.Recommended.Admission.Plugins)
shootdnshostedzone.Register(o.Recommended.Admission.Plugins)
shootvalidator.Register(o.Recommended.Admission.Plugins)
deletionconfirmation.Register(o.Recommended.Admission.Plugins)

allOrderedPlugins := []string{
resourcereferencemanager.PluginName,
deletionconfirmation.PluginName,
shootdnshostedzone.PluginName,
shootquotavalidator.PluginName,
shootseedmanager.PluginName,
Expand All @@ -129,20 +132,13 @@ func (o *Options) complete() error {
recommendedPluginOrder.Insert(allOrderedPlugins...)
o.Recommended.Admission.RecommendedPluginOrder = recommendedPluginOrder.List()

// TODO: Temporary deletionconfirmation it as disabled by default.
// Enable it once the old shoot deletion mechanics is removed.
o.Recommended.Admission.DefaultOffPlugins.Insert(deletionconfirmation.PluginName)
return nil
}

func (o *Options) config() (*apiserver.Config, error) {
// Enable some admission plugins by default
enabledPlugins := sets.NewString(o.Recommended.Admission.EnablePlugins...)
if !enabledPlugins.Has(resourcereferencemanager.PluginName) {
enabledPlugins.Insert(resourcereferencemanager.PluginName)
}
if !enabledPlugins.Has(shootvalidator.PluginName) {
enabledPlugins.Insert(shootvalidator.PluginName)
}
o.Recommended.Admission.EnablePlugins = enabledPlugins.List()

// Create clientset for the garden.sapcloud.io API group
// Use loopback config to create a new Kubernetes client for the garden.sapcloud.io API group
gardenerAPIServerConfig := genericapiserver.NewRecommendedConfig(api.Codecs)
Expand Down
1 change: 1 addition & 0 deletions hack/delete-shoot
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
NAME="$1"
NAMESPACE="${2:-$(id -u -n)}"

kubectl --namespace "$NAMESPACE" patch shoot "$NAME" --type=merge -p "{\"metadata\": {\"annotations\": {\"confirmation.garden.sapcloud.io/deletion\": \"true\"}}}"
kubectl --namespace "$NAMESPACE" delete shoot "$NAME"
deletionTimestamp="$(kubectl --namespace "$NAMESPACE" get shoot "$NAME" -o jsonpath --template={.metadata.deletionTimestamp})"
kubectl --namespace "$NAMESPACE" patch shoot "$NAME" --type=merge -p "{\"metadata\": {\"annotations\": {\"confirmation.garden.sapcloud.io/deletionTimestamp\": \"$deletionTimestamp\"}}}"
5 changes: 5 additions & 0 deletions pkg/operation/common/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,13 @@ const (

// ConfirmationDeletionTimestamp is an annotation on a Shoot resource whose value must be set equal to the Shoot's
// '.metadata.deletionTimestamp' value to trigger the deletion process of the Shoot cluster.
// DEPRECATED: Use ConfirmationDeletion instead.
ConfirmationDeletionTimestamp = "confirmation.garden.sapcloud.io/deletionTimestamp"

// ConfirmationDeletion is an annotation on a Shoot resource whose value must be set to true
// to trigger the deletion process of the Shoot cluster.
ConfirmationDeletion = "confirmation.garden.sapcloud.io/deletion"

// ControllerManagerInternalConfigMapName is the name of the internal config map in which the Gardener controller
// manager stores its configuration.
ControllerManagerInternalConfigMapName = "gardener-controller-manager-internal-config"
Expand Down
117 changes: 117 additions & 0 deletions plugin/pkg/global/deletionconfirmation/admission.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deletionconfirmation

import (
"errors"
"fmt"
"io"
"strconv"

"github.com/gardener/gardener/pkg/apis/garden"
admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer"
gardeninformers "github.com/gardener/gardener/pkg/client/garden/informers/internalversion"
gardenlisters "github.com/gardener/gardener/pkg/client/garden/listers/garden/internalversion"
"github.com/gardener/gardener/pkg/operation/common"
"k8s.io/apiserver/pkg/admission"
)

const (
// PluginName is the name of this admission plugin.
PluginName = "DeletionConfirmation"
)

// Register registers a plugin.
func Register(plugins *admission.Plugins) {
plugins.Register(PluginName, NewFactory)
}

// NewFactory creates a new PluginFactory.
func NewFactory(config io.Reader) (admission.Interface, error) {
return New()
}

// WaitFunc is a function that waits for true.
type WaitFunc func() bool

// DeletionConfirmation contains an admission handler and listers.
type DeletionConfirmation struct {
*admission.Handler
shootLister gardenlisters.ShootLister
waitFunc WaitFunc
}

var _ = admissioninitializer.WantsInternalGardenInformerFactory(&DeletionConfirmation{})

// New creates a new DeletionConfirmation admission plugin.
func New() (*DeletionConfirmation, error) {
h := admission.NewHandler(admission.Delete)
return &DeletionConfirmation{
Handler: h,
waitFunc: h.WaitForReady,
}, nil
}

// SetInternalGardenInformerFactory gets Lister from SharedInformerFactory.
func (d *DeletionConfirmation) SetInternalGardenInformerFactory(f gardeninformers.SharedInformerFactory) {
d.shootLister = f.Garden().InternalVersion().Shoots().Lister()
}

// SetWaitFunc sets the wait function
func (d *DeletionConfirmation) SetWaitFunc(w WaitFunc) {
d.waitFunc = w
}

// ValidateInitialization checks whether the plugin was correctly initialized.
func (d *DeletionConfirmation) ValidateInitialization() error {
if d.shootLister == nil {
return errors.New("missing shoot lister")
}
return nil
}

// Admit makes admissions decisions based on deletion confirmation annotation.
func (d *DeletionConfirmation) Admit(a admission.Attributes) error {
// Ignore all kinds other than Shoot.
// TODO: in future the Kinds should be configurable
// https://v1-9.docs.kubernetes.io/docs/admin/admission-controllers/#imagepolicywebhook
if a.GetKind().GroupKind() != garden.Kind("Shoot") {
return nil
}

// Wait until the caches have been synced
if !d.waitFunc() {
return admission.NewForbidden(a, errors.New("not yet ready to handle request"))
}

shoot, err := d.shootLister.Shoots(a.GetNamespace()).Get(a.GetName())
if err != nil {
return err
}

annotations := shoot.GetAnnotations()

if annotations == nil {
return admission.NewForbidden(a, annotationRequiredError())
}
if present, _ := strconv.ParseBool(annotations[common.ConfirmationDeletion]); !present {
return admission.NewForbidden(a, annotationRequiredError())
}
return nil
}

func annotationRequiredError() error {
return fmt.Errorf("must have a %q annotation to delete", common.ConfirmationDeletion)
}
172 changes: 172 additions & 0 deletions plugin/pkg/global/deletionconfirmation/admission_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deletionconfirmation_test

import (
. "github.com/gardener/gardener/plugin/pkg/global/deletionconfirmation"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"github.com/gardener/gardener/pkg/apis/garden"
gardeninformers "github.com/gardener/gardener/pkg/client/garden/informers/internalversion"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/tools/cache"
)

func ready() bool {
return true
}

func notReady() bool {
return false
}

var _ = Describe("deleteconfirmation", func() {

Describe("#Admit", func() {
var (
shoot garden.Shoot
attrs admission.Attributes
shootStore cache.Store
admissionHandler *DeletionConfirmation
gardenInformerFactory gardeninformers.SharedInformerFactory
)

BeforeEach(func() {
admissionHandler, _ = New()
gardenInformerFactory = gardeninformers.NewSharedInformerFactory(nil, 0)
admissionHandler.SetInternalGardenInformerFactory(gardenInformerFactory)
shootStore = gardenInformerFactory.Garden().InternalVersion().Shoots().Informer().GetStore()
admissionHandler.SetReadyFunc(ready)
shoot = garden.Shoot{
ObjectMeta: metav1.ObjectMeta{
Name: "dummy",
Namespace: "dummy",
},
}
attrs = admission.NewAttributesRecord(nil, nil, garden.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Namespace, garden.Resource("shoots").WithVersion("version"), "", admission.Delete, nil)
})

It("should do nothing because the resource is not Shoot", func() {
attrs = admission.NewAttributesRecord(nil, nil, garden.Kind("Foo").WithVersion("version"), shoot.Namespace, shoot.Name, garden.Resource("foos").WithVersion("version"), "", admission.Delete, nil)

err := admissionHandler.Admit(attrs)

Expect(err).NotTo(HaveOccurred())
})

It("should do nothing because cache is not ready", func() {
admissionHandler.SetWaitFunc(notReady)
err := admissionHandler.Admit(attrs)

Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("not yet ready to handle request"))
})

It("should do nothing because the resource is already removed", func() {
err := admissionHandler.Admit(attrs)

Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal(`shoot.garden.sapcloud.io "dummy" not found`))
})

Context("no annotation", func() {
It("should reject for nil annotation field", func() {
shootStore.Add(&shoot)

err := admissionHandler.Admit(attrs)

Expect(err).To(HaveOccurred())
Expect(apierrors.IsForbidden(err)).To(BeTrue())
})

It("should reject for false annotation value", func() {
shoot.Annotations = map[string]string{
"confirmation.garden.sapcloud.io/deletion": "false",
}
shootStore.Add(&shoot)

err := admissionHandler.Admit(attrs)

Expect(err).To(HaveOccurred())
Expect(apierrors.IsForbidden(err)).To(BeTrue())
})

It("should succeed for true annotation value", func() {
shoot.Annotations = map[string]string{
"confirmation.garden.sapcloud.io/deletion": "true",
}
shootStore.Add(&shoot)

err := admissionHandler.Admit(attrs)

Expect(err).NotTo(HaveOccurred())
})
})

})
Describe("#Register", func() {
It("should register the plugin", func() {
plugins := admission.NewPlugins()
Register(plugins)

registred := plugins.Registered()
Expect(registred).To(HaveLen(1))
Expect(registred).To(ContainElement("DeletionConfirmation"))
})
})

Describe("#NewFactory", func() {
It("should create a new PluginFactory", func() {
f, err := NewFactory(nil)

Expect(f).NotTo(BeNil())
Expect(err).ToNot(HaveOccurred())
})
})

Describe("#New", func() {
It("should only handle DELETE operations", func() {
dr, err := New()

Expect(err).ToNot(HaveOccurred())
Expect(dr.Handles(admission.Create)).NotTo(BeTrue())
Expect(dr.Handles(admission.Update)).NotTo(BeTrue())
Expect(dr.Handles(admission.Connect)).NotTo(BeTrue())
Expect(dr.Handles(admission.Delete)).To(BeTrue())
})
})

Describe("#ValidateInitialization", func() {
It("should return error if no ShootLister is set", func() {
dr, _ := New()

err := dr.ValidateInitialization()

Expect(err).To(HaveOccurred())
})

It("should not return error if ShootLister is set", func() {
dr, _ := New()
dr.SetInternalGardenInformerFactory(gardeninformers.NewSharedInformerFactory(nil, 0))

err := dr.ValidateInitialization()

Expect(err).ToNot(HaveOccurred())
})
})
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deletionconfirmation_test

import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"testing"
)

func TestDeletionConfirmation(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Admission DeletionConfirmation Suite")
}

0 comments on commit cce0385

Please sign in to comment.