From 060e0dd815ca50b771b173439c96ae9c97b32ff7 Mon Sep 17 00:00:00 2001 From: erikfu Date: Mon, 26 Feb 2024 15:09:22 -0800 Subject: [PATCH 01/13] Adding core webhook libraries sourced from aws lb controller --- go.mod | 2 +- pkg/webhook/core/context.go | 23 ++ pkg/webhook/core/context_test.go | 53 +++ pkg/webhook/core/mutating_handler.go | 83 +++++ pkg/webhook/core/mutating_handler_test.go | 399 ++++++++++++++++++++++ pkg/webhook/core/mutator.go | 28 ++ pkg/webhook/core/mutator_mocks.go | 82 +++++ 7 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 pkg/webhook/core/context.go create mode 100644 pkg/webhook/core/context_test.go create mode 100644 pkg/webhook/core/mutating_handler.go create mode 100644 pkg/webhook/core/mutating_handler_test.go create mode 100644 pkg/webhook/core/mutator.go create mode 100644 pkg/webhook/core/mutator_mocks.go diff --git a/go.mod b/go.mod index acf705a7..5d506c63 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + gomodules.xyz/jsonpatch/v2 v2.4.0 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -65,7 +66,6 @@ require ( golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/pkg/webhook/core/context.go b/pkg/webhook/core/context.go new file mode 100644 index 00000000..560ae7a1 --- /dev/null +++ b/pkg/webhook/core/context.go @@ -0,0 +1,23 @@ +package core + +import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type contextKey string + +const ( + contextKeyAdmissionRequest contextKey = "admissionRequest" +) + +func ContextGetAdmissionRequest(ctx context.Context) *admission.Request { + if v := ctx.Value(contextKeyAdmissionRequest); v != nil { + return v.(*admission.Request) + } + return nil +} + +func ContextWithAdmissionRequest(ctx context.Context, req admission.Request) context.Context { + return context.WithValue(ctx, contextKeyAdmissionRequest, &req) +} diff --git a/pkg/webhook/core/context_test.go b/pkg/webhook/core/context_test.go new file mode 100644 index 00000000..5be7f3d2 --- /dev/null +++ b/pkg/webhook/core/context_test.go @@ -0,0 +1,53 @@ +package core + +import ( + "context" + "github.com/stretchr/testify/assert" + admissionv1 "k8s.io/api/admission/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "testing" +) + +func TestContextGetAdmissionRequestAndContextWithAdmissionRequest(t *testing.T) { + type args struct { + req *admission.Request + } + tests := []struct { + name string + args args + want *admission.Request + }{ + { + name: "with request", + args: args{ + req: &admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "1", + }, + }, + }, + want: &admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "1", + }, + }, + }, + { + name: "without request", + args: args{ + req: nil, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + if tt.args.req != nil { + ctx = ContextWithAdmissionRequest(ctx, *tt.args.req) + } + got := ContextGetAdmissionRequest(ctx) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/webhook/core/mutating_handler.go b/pkg/webhook/core/mutating_handler.go new file mode 100644 index 00000000..dca280d2 --- /dev/null +++ b/pkg/webhook/core/mutating_handler.go @@ -0,0 +1,83 @@ +package core + +import ( + "context" + "encoding/json" + admissionv1 "k8s.io/api/admission/v1" + "net/http" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var mutatingHandlerLog = ctrl.Log.WithName("mutating_handler") + +type mutatingHandler struct { + mutator Mutator + decoder *admission.Decoder +} + +func (h *mutatingHandler) SetDecoder(d *admission.Decoder) { + h.decoder = d +} + +// Handle handles admission requests. +func (h *mutatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + mutatingHandlerLog.V(1).Info("mutating webhook request", "request", req) + var resp admission.Response + switch req.Operation { + case admissionv1.Create: + resp = h.handleCreate(ctx, req) + case admissionv1.Update: + resp = h.handleUpdate(ctx, req) + default: + resp = admission.Allowed("") + } + mutatingHandlerLog.V(1).Info("mutating webhook response", "response", resp) + return resp +} + +func (h *mutatingHandler) handleCreate(ctx context.Context, req admission.Request) admission.Response { + prototype, err := h.mutator.Prototype(req) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + obj := prototype.DeepCopyObject() + if err := h.decoder.DecodeRaw(req.Object, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + mutatedObj, err := h.mutator.MutateCreate(ContextWithAdmissionRequest(ctx, req), obj) + if err != nil { + return admission.Denied(err.Error()) + } + mutatedObjPayload, err := json.Marshal(mutatedObj) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, mutatedObjPayload) +} + +func (h *mutatingHandler) handleUpdate(ctx context.Context, req admission.Request) admission.Response { + prototype, err := h.mutator.Prototype(req) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + obj := prototype.DeepCopyObject() + oldObj := prototype.DeepCopyObject() + if err := h.decoder.DecodeRaw(req.Object, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if err := h.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + mutatedObj, err := h.mutator.MutateUpdate(ContextWithAdmissionRequest(ctx, req), obj, oldObj) + if err != nil { + return admission.Denied(err.Error()) + } + mutatedObjPayload, err := json.Marshal(mutatedObj) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, mutatedObjPayload) +} diff --git a/pkg/webhook/core/mutating_handler_test.go b/pkg/webhook/core/mutating_handler_test.go new file mode 100644 index 00000000..5f09255d --- /dev/null +++ b/pkg/webhook/core/mutating_handler_test.go @@ -0,0 +1,399 @@ +package core + +import ( + "context" + "encoding/json" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "testing" +) + +func Test_mutatingHandler_InjectDecoder(t *testing.T) { + h := mutatingHandler{ + decoder: nil, + } + decoder := &admission.Decoder{} + h.SetDecoder(decoder) + + assert.Equal(t, decoder, h.decoder) +} + +func Test_mutatingHandler_Handle(t *testing.T) { + schema := runtime.NewScheme() + clientgoscheme.AddToScheme(schema) + // k8sDecoder knows k8s objects + decoder := admission.NewDecoder(schema) + patchTypeJSONPatch := admissionv1.PatchTypeJSONPatch + + initialPod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + Annotations: map[string]string{ + "some-key": "some-value", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "bar", + Image: "bar:v1", + }, + }, + }, + Status: corev1.PodStatus{}, + } + initialPodRaw, err := json.Marshal(initialPod) + assert.NoError(t, err) + updatedPod := initialPod.DeepCopy() + updatedPod.Spec.Containers[0].Image = "bar:v2" + updatedPodRaw, err := json.Marshal(updatedPod) + assert.NoError(t, err) + + type fields struct { + mutatorPrototype func(req admission.Request) (runtime.Object, error) + mutatorMutateCreate func(ctx context.Context, obj runtime.Object) (runtime.Object, error) + mutatorMutateUpdate func(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) + decoder *admission.Decoder + } + type args struct { + req admission.Request + } + tests := []struct { + name string + fields fields + args args + want admission.Response + }{ + { + name: "[create] approve request and mutates nothing", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateCreate: func(ctx context.Context, obj runtime.Object) (runtime.Object, error) { + return obj, nil + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: []jsonpatch.JsonPatchOperation{}, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + PatchType: nil, + }, + }, + }, + { + name: "[create] approve request and mutates object annotations", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateCreate: func(ctx context.Context, obj runtime.Object) (runtime.Object, error) { + pod := obj.(*corev1.Pod) + pod.Annotations["some-name"] = "some-value" + return pod, nil + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: []jsonpatch.JsonPatchOperation{ + { + Operation: "add", + Path: "/metadata/annotations/some-name", + Value: "some-value", + }, + }, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + PatchType: &patchTypeJSONPatch, + }, + }, + }, + { + name: "[create] reject request", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateCreate: func(ctx context.Context, obj runtime.Object) (runtime.Object, error) { + return nil, errors.New("oops, some error happened") + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: nil, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: http.StatusForbidden, + Message: "oops, some error happened", + Reason: "Forbidden", + }, + }, + }, + }, + { + name: "[create] unexpected object type - prototype returns error ", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return nil, errors.New("oops, unexpected object type") + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: nil, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: http.StatusBadRequest, + Message: "oops, unexpected object type", + }, + }, + }, + }, + { + name: "[update] approve request and mutates nothing", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateUpdate: func(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { + return obj, nil + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: updatedPodRaw, + }, + OldObject: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: []jsonpatch.JsonPatchOperation{}, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + PatchType: nil, + }, + }, + }, + { + name: "[update] approve request and mutates object annotations", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateUpdate: func(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { + pod := obj.(*corev1.Pod) + pod.Annotations["some-name"] = "some-value" + return pod, nil + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: updatedPodRaw, + }, + OldObject: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: []jsonpatch.JsonPatchOperation{ + { + Operation: "add", + Path: "/metadata/annotations/some-name", + Value: "some-value", + }, + }, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + PatchType: &patchTypeJSONPatch, + }, + }, + }, + { + name: "[update] reject request", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil + }, + mutatorMutateUpdate: func(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { + return nil, errors.New("oops, some error happened") + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: updatedPodRaw, + }, + OldObject: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: nil, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: http.StatusForbidden, + Message: "oops, some error happened", + Reason: "Forbidden", + }, + }, + }, + }, + { + name: "[update] unexpected object type - prototype returns error ", + fields: fields{ + mutatorPrototype: func(req admission.Request) (runtime.Object, error) { + return nil, errors.New("oops, unexpected object type") + }, + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: updatedPodRaw, + }, + OldObject: runtime.RawExtension{ + Raw: initialPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: nil, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: http.StatusBadRequest, + Message: "oops, unexpected object type", + }, + }, + }, + }, + { + name: "[connect] methods other than create/update will pass through", + fields: fields{ + decoder: decoder, + }, + args: args{ + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Connect, + Object: runtime.RawExtension{ + Raw: updatedPodRaw, + }, + }, + }, + }, + want: admission.Response{ + Patches: nil, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Code: http.StatusOK, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mutator := NewMockMutator(ctrl) + if tt.fields.mutatorPrototype != nil { + mutator.EXPECT().Prototype(gomock.Any()).DoAndReturn(tt.fields.mutatorPrototype) + } + if tt.fields.mutatorMutateCreate != nil { + mutator.EXPECT().MutateCreate(gomock.Any(), gomock.Any()).DoAndReturn(tt.fields.mutatorMutateCreate) + } + if tt.fields.mutatorMutateUpdate != nil { + mutator.EXPECT().MutateUpdate(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(tt.fields.mutatorMutateUpdate) + } + + h := &mutatingHandler{ + mutator: mutator, + decoder: tt.fields.decoder, + } + got := h.Handle(ctx, tt.args.req) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/webhook/core/mutator.go b/pkg/webhook/core/mutator.go new file mode 100644 index 00000000..2b39750c --- /dev/null +++ b/pkg/webhook/core/mutator.go @@ -0,0 +1,28 @@ +package core + +import ( + "context" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +//go:generate mockgen -destination mutator_mocks.go -package core github.com/aws/aws-application-networking-k8s/pkg/webhook/core Mutator +type Mutator interface { + // Prototype returns a prototype of Object for this admission request. + Prototype(req admission.Request) (runtime.Object, error) + + // MutateCreate handles Object creation and returns the object after mutation and error if any. + MutateCreate(ctx context.Context, obj runtime.Object) (runtime.Object, error) + // MutateUpdate handles Object update and returns the object after mutation and error if any. + MutateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) +} + +// MutatingWebhookForMutator creates a new mutating Webhook. +func MutatingWebhookForMutator(scheme *runtime.Scheme, mutator Mutator) *admission.Webhook { + return &admission.Webhook{ + Handler: &mutatingHandler{ + mutator: mutator, + decoder: admission.NewDecoder(scheme), + }, + } +} diff --git a/pkg/webhook/core/mutator_mocks.go b/pkg/webhook/core/mutator_mocks.go new file mode 100644 index 00000000..03ec4ed8 --- /dev/null +++ b/pkg/webhook/core/mutator_mocks.go @@ -0,0 +1,82 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/aws/aws-application-networking-k8s/pkg/webhook/core (interfaces: Mutator) + +// Package core is a generated GoMock package. +package core + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + runtime "k8s.io/apimachinery/pkg/runtime" + admission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// MockMutator is a mock of Mutator interface. +type MockMutator struct { + ctrl *gomock.Controller + recorder *MockMutatorMockRecorder +} + +// MockMutatorMockRecorder is the mock recorder for MockMutator. +type MockMutatorMockRecorder struct { + mock *MockMutator +} + +// NewMockMutator creates a new mock instance. +func NewMockMutator(ctrl *gomock.Controller) *MockMutator { + mock := &MockMutator{ctrl: ctrl} + mock.recorder = &MockMutatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMutator) EXPECT() *MockMutatorMockRecorder { + return m.recorder +} + +// MutateCreate mocks base method. +func (m *MockMutator) MutateCreate(arg0 context.Context, arg1 runtime.Object) (runtime.Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MutateCreate", arg0, arg1) + ret0, _ := ret[0].(runtime.Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MutateCreate indicates an expected call of MutateCreate. +func (mr *MockMutatorMockRecorder) MutateCreate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MutateCreate", reflect.TypeOf((*MockMutator)(nil).MutateCreate), arg0, arg1) +} + +// MutateUpdate mocks base method. +func (m *MockMutator) MutateUpdate(arg0 context.Context, arg1, arg2 runtime.Object) (runtime.Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MutateUpdate", arg0, arg1, arg2) + ret0, _ := ret[0].(runtime.Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MutateUpdate indicates an expected call of MutateUpdate. +func (mr *MockMutatorMockRecorder) MutateUpdate(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MutateUpdate", reflect.TypeOf((*MockMutator)(nil).MutateUpdate), arg0, arg1, arg2) +} + +// Prototype mocks base method. +func (m *MockMutator) Prototype(arg0 admission.Request) (runtime.Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Prototype", arg0) + ret0, _ := ret[0].(runtime.Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Prototype indicates an expected call of Prototype. +func (mr *MockMutatorMockRecorder) Prototype(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prototype", reflect.TypeOf((*MockMutator)(nil).Prototype), arg0) +} From 38fc75b7c3a41857523bdff8e4f4996c4da04850 Mon Sep 17 00:00:00 2001 From: erikfu Date: Mon, 26 Feb 2024 16:39:32 -0800 Subject: [PATCH 02/13] Inject pod readiness gate via pod mutating webhook --- Makefile | 17 +++ cmd/aws-application-networking-k8s/main.go | 31 ++++- config/default/kustomization.yaml | 63 +-------- config/manager/kustomization.yaml | 3 +- config/manager/manager.yaml | 9 ++ config/prometheus/kustomization.yaml | 2 + config/rbac/kustomization.yaml | 2 + config/webhook/kustomization.yaml | 4 + config/webhook/manifests.yaml | 53 ++++++++ docs/guides/pod-readiness-gates.md | 128 ++++++++++++++++++ pkg/config/controller_config.go | 9 +- pkg/webhook/pod_mutator.go | 50 +++++++ pkg/webhook/pod_mutator_test.go | 81 +++++++++++ pkg/webhook/pod_readiness_gate_injector.go | 43 ++++++ .../webhook/readiness_gate_inject_test.go | 85 ++++++++++++ test/suites/webhook/suite_test.go | 40 ++++++ 16 files changed, 548 insertions(+), 72 deletions(-) create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 docs/guides/pod-readiness-gates.md create mode 100644 pkg/webhook/pod_mutator.go create mode 100644 pkg/webhook/pod_mutator_test.go create mode 100644 pkg/webhook/pod_readiness_gate_injector.go create mode 100644 test/suites/webhook/readiness_gate_inject_test.go create mode 100644 test/suites/webhook/suite_test.go diff --git a/Makefile b/Makefile index 0d047a9d..211fdcfc 100644 --- a/Makefile +++ b/Makefile @@ -144,3 +144,20 @@ api-reference: ## Update documentation in docs/api-reference.md docs: mkdir -p site mkdocs build + +# NB webhook tests can only run if the controller is deployed to the cluster +webhook-e2e-test-namespace := "webhook-e2e-test" + +.PHONY: webhook-e2e-test +webhook-e2e-test: + @kubectl create namespace $(webhook-e2e-test-namespace) > /dev/null 2>&1 || true # ignore already exists error + LOG_LEVEL=debug + cd test && go test \ + -p 1 \ + -count 1 \ + -timeout 10m \ + -v \ + ./suites/webhook/... \ + --ginkgo.focus="${FOCUS}" \ + --ginkgo.skip="${SKIP}" \ + --ginkgo.v \ No newline at end of file diff --git a/cmd/aws-application-networking-k8s/main.go b/cmd/aws-application-networking-k8s/main.go index 6ee85ee3..84848b2c 100644 --- a/cmd/aws-application-networking-k8s/main.go +++ b/cmd/aws-application-networking-k8s/main.go @@ -18,10 +18,11 @@ package main import ( "flag" - "os" - + "github.com/aws/aws-application-networking-k8s/pkg/webhook" "github.com/go-logr/zapr" "go.uber.org/zap/zapcore" + "os" + k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -49,7 +50,6 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/k8s" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) var ( @@ -128,14 +128,24 @@ func main() { setupLog.Fatal("cloud client setup failed: %s", err) } + // do not create the webhook server when running locally + var webhookServer k8swebhook.Server + isLocalDev := config.DevMode != "" + if !isLocalDev { + webhookServer = k8swebhook.NewServer(k8swebhook.Options{ + Port: 9443, + CertDir: "/etc/webhook-cert/", + CertName: "tls.crt", + KeyName: "tls.key", + }) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr, }, - WebhookServer: webhook.NewServer(webhook.Options{ - Port: 9443, - }), + WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "amazon-vpc-lattice.io", @@ -144,6 +154,15 @@ func main() { setupLog.Fatal("manager setup failed:", err) } + if !isLocalDev { + // register webhook handlers + readinessGateInjector := webhook.NewPodReadinessGateInjector( + mgr.GetClient(), + log.Named("pod-readiness-gate-injector"), + ) + webhook.NewPodMutator(scheme, readinessGateInjector).SetupWithManager(mgr) + } + finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient()) // parent logging scope for all controllers diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index b542098c..3acfe15b 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,26 +1,10 @@ -# Adds namespace to all resources. -#namespace: code-aws-application-networking-system - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -#namePrefix: code- - -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue - -bases: +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: - ../crds - ../rbac - ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -33,42 +17,3 @@ patchesStrategicMerge: # Mount the controller config file for loading manager configurations # through a ComponentConfig type #- manager_config_patch.yaml - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 9b968479..2cd86eed 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,4 @@ configMapGenerator: name: manager-config images: - name: controller - newName: public.ecr.aws/aws-application-networking-k8s/aws-gateway-controller - newTag: v1.0.3 + newName: 671123684841.dkr.ecr.us-west-2.amazonaws.com/controller diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c2995bbc..0928f194 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -60,5 +60,14 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - mountPath: /etc/webhook-cert + name: webhook-cert + readOnly: true serviceAccountName: gateway-api-controller terminationGracePeriodSeconds: 10 + volumes: + - name: webhook-cert + secret: + defaultMode: 420 + secretName: webhook-cert \ No newline at end of file diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed137168..7f6a57fb 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - monitor.yaml diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index f0549f47..7f06e49a 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: # All RBAC will be applied under this service account in # the deployment namespace. You may comment out this resource diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..cd08260e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - manifests.yaml \ No newline at end of file diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 00000000..2b5432b8 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,53 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: aws-appnet-gwc-mutating-webhook +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + # placeholder newline for caBundle, can replace with actual CA before install using yq + # or afterward with a patch on the webhook + caBundle: Cg== + service: + name: webhook-service + namespace: aws-application-networking-system + path: /mutate-pod + failurePolicy: Fail + name: mpod.gwc.k8s.aws + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + namespaceSelector: + matchExpressions: + - key: aws-application-networking-k8s/pod-readiness-gate-inject + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - gateway-api-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: aws-application-networking-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: gateway-api-controller \ No newline at end of file diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md new file mode 100644 index 00000000..76130336 --- /dev/null +++ b/docs/guides/pod-readiness-gates.md @@ -0,0 +1,128 @@ +# Pod readiness gate + +# TODO: Update to reflect final implementation of readiness gate logic. Replace + +AWS Gateway API controller supports [»Pod readiness gates«](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate) to indicate that pod is registered to the VPC Lattice and healthy to receive traffic. +The controller automatically injects the necessary readiness gate configuration to the pod spec via mutating webhook during pod creation. + +For readiness gate configuration to be injected to the pod spec, you need to apply the label `aws-application-networking-k8s/pod-readiness-gate-inject: enabled` to the pod namespace. + +The pod readiness gate is needed under certain circumstances to achieve full zero downtime rolling deployments. Consider the following example: + +* Low number of replicas in a deployment +* Start a rolling update of the deployment +* Rollout of new pods takes less time than it takes the AWS Gateway API controller to register the new pods and for their health state turn »Healthy« in the target group +* At some point during this rolling update, the target group might only have registered targets that are in »Initial« or »Draining« state; this results in service outage + +In order to avoid this situation, the AWS Gateway API controller can set the readiness condition on the pods that constitute your ingress or service backend. The condition status on a pod will be set to `True` only when the corresponding target in the VPC Lattice target group shows a health state of »Healthy«. +This prevents the rolling update of a deployment from terminating old pods until the newly created pods are »Healthy« in the VPC Lattice target group and ready to take traffic. + +## Setup +Pod readiness gates rely on [»admission webhooks«](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), where the Kubernetes API server makes calls to the AWS Gateway API controller as part of pod creation. This call is made using TLS, so the controller must present a TLS certificate. This certificate is stored as a standard Kubernetes secret. If you are using Helm, the certificate will automatically be configured as part of the Helm install. + +If you are manually deploying the controller using the ```deploy.yaml``` file, you will need to also create the tls secret in the controller namespace. + +```console +CERT_FILE=tls.crt +KEY_FILE=tls.key + +WEBHOOK_SVC_NAME=webhook-service +WEBHOOK_NAME=aws-appnet-gwc-mutating-webhook +WEBHOOK_NAMESPACE=aws-application-networking-system +WEBHOOK_SECRET_NAME=webhook-cert + +# Step 1: generate a certificate if needed, can also be provisioned through orgnanizational PKI, etc +# This cert includes a 100 year expiry +HOST=${WEBHOOK_SVC_NAME}.${WEBHOOK_NAMESPACE}.svc +openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" \ + -addext "subjectAltName = DNS:${HOST}, DNS:${HOST}.cluster.local" + +# Step 2: add the secret - can be done so long as the namespace exists +# note that running "kubectl delete -f deploy.yaml" will remove the controller namespace AND this secret. +kubectl create secret tls $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE --cert=${CERT_FILE} --key=${KEY_FILE} + +# Step 3: after applying deploy.yaml, patch the webhook CA bundle to exactly the cert being used +# this will ensure Kubernetes API server is able to trust the certificate presented by the webhook +CERT_B64=$(cat tls.crt | base64) +kubectl patch mutatingwebhookconfigurations.admissionregistration.k8s.io $WEBHOOK_NAME \ + --namespace $WEBHOOK_NAMESPACE --type='json' \ + -p="[{'op': 'replace', 'path': '/webhooks/0/clientConfig/caBundle', 'value': '${CERT_B64}'}]" +``` + +## Configuration +Pod readiness gate support is enabled by default on the AWS Gateway API controller. To enable the feature, you must apply a label to each of the namespaces you would like to use this feature. You can create and label a namespace as follows - + +``` +$ kubectl create namespace example-ns +namespace/example-ns created + +$ kubectl label namespace example-ns aws-application-networking-k8s/pod-readiness-gate-inject=enabled +namespace/example-ns labeled + +$ kubectl describe namespace example-ns +Name: example-ns +Labels: aws-application-networking-k8s/pod-readiness-gate-inject=enabled + kubernetes.io/metadata.name=example-ns +Annotations: +Status: Active +``` + +Once labelled, the controller will add the pod readiness gates to all subsequently created pods. + +The readiness gates have the prefix `` and the controller injects the config to the pod spec only during pod creation. + +## Object Selector +The default webhook configuration matches all pods in the namespaces containing the label `aws-application-networking-k8s/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select ONLY pods with `aws-application-networking-k8s/pod-readiness-gate-inject: enabled` label instead of all pods in the labeled namespace, you can add the following `objectSelector` to the webhook: +``` + objectSelector: + matchLabels: + aws-application-networking-k8s/pod-readiness-gate-inject: enabled +``` +To edit, +``` +$ kubectl edit mutatingwebhookconfigurations aws-appnet-gwc-mutating-webhook + ... + name: mpod.gwc.k8s.aws + namespaceSelector: + matchExpressions: + - key: aws-application-networking-k8s/pod-readiness-gate-inject + operator: In + values: + - enabled + objectSelector: + matchLabels: + aws-application-networking-k8s/pod-readiness-gate-inject: enabled + ... +``` +When you specify multiple selectors, pods matching all the conditions will get mutated. + +## Disabling the readiness gate inject +You can specify the controller flag `--enable-pod-readiness-gate-inject=false` during controller startup to disable the controller from modifying the pod spec. + +## Checking the pod condition status + +The status of the readiness gates can be verified with `kubectl get pod -o wide`: +``` +NAME READY STATUS RESTARTS AGE IP NODE READINESS GATES +nginx-test-5744b9ff84-7ftl9 1/1 Running 0 81s 10.1.2.3 ip-10-1-2-3.ec2.internal 0/1 +``` + +When the target is registered and healthy in the VPC Lattice target group, the output will look like: +``` +NAME READY STATUS RESTARTS AGE IP NODE READINESS GATES +nginx-test-5744b9ff84-7ftl9 1/1 Running 0 81s 10.1.2.3 ip-10-1-2-3.ec2.internal 1/1 +``` + +If a readiness gate doesn't get ready, you can check the reason via: + +```console +$ kubectl get pod nginx-test-545d8f4d89-l7rcl -o yaml | grep -B7 'type: ' +status: + conditions: + - lastProbeTime: null + lastTransitionTime: null + message: + reason: HEALTHY + status: "True" + type: +``` diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 07a53b28..133bd3a6 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -25,6 +25,7 @@ const ( DEFAULT_SERVICE_NETWORK = "DEFAULT_SERVICE_NETWORK" ENABLE_SERVICE_NETWORK_OVERRIDE = "ENABLE_SERVICE_NETWORK_OVERRIDE" AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" + DEV_MODE = "DEV_MODE" ) var VpcID = "" @@ -32,6 +33,7 @@ var AccountID = "" var Region = "" var DefaultServiceNetwork = "" var ClusterName = "" +var DevMode = "" var ServiceNetworkOverrideMode = false @@ -44,7 +46,8 @@ func ConfigInit() error { func configInit(sess *session.Session, metadata EC2Metadata) error { var err error - // CLUSTER_VPC_ID + DevMode = os.Getenv(DEV_MODE) + VpcID = os.Getenv(CLUSTER_VPC_ID) if VpcID == "" { VpcID, err = metadata.VpcID() @@ -53,7 +56,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // REGION Region = os.Getenv(REGION) if Region == "" { Region, err = metadata.Region() @@ -62,7 +64,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // AWS_ACCOUNT_ID AccountID = os.Getenv(AWS_ACCOUNT_ID) if AccountID == "" { AccountID, err = metadata.AccountId() @@ -71,7 +72,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // DEFAULT_SERVICE_NETWORK DefaultServiceNetwork = os.Getenv(DEFAULT_SERVICE_NETWORK) overrideFlag := os.Getenv(ENABLE_SERVICE_NETWORK_OVERRIDE) @@ -79,7 +79,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { ServiceNetworkOverrideMode = true } - // CLUSTER_NAME ClusterName, err = getClusterName(sess) if err != nil { return fmt.Errorf("cannot get cluster name: %s", err) diff --git a/pkg/webhook/pod_mutator.go b/pkg/webhook/pod_mutator.go new file mode 100644 index 00000000..99826f47 --- /dev/null +++ b/pkg/webhook/pod_mutator.go @@ -0,0 +1,50 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/webhook/core" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + apiPathMutatePod = "/mutate-pod" +) + +func NewPodMutator(scheme *runtime.Scheme, podReadinessGateInjector *PodReadinessGateInjector) *podMutator { + return &podMutator{ + podReadinessGateInjector: podReadinessGateInjector, + scheme: scheme, + } +} + +var _ core.Mutator = &podMutator{} + +type podMutator struct { + podReadinessGateInjector *PodReadinessGateInjector + scheme *runtime.Scheme +} + +func (m *podMutator) Prototype(_ admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil +} + +func (m *podMutator) MutateCreate(ctx context.Context, obj runtime.Object) (runtime.Object, error) { + pod := obj.(*corev1.Pod) + if err := m.podReadinessGateInjector.Mutate(ctx, pod); err != nil { + return pod, err + } + return pod, nil +} + +func (m *podMutator) MutateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { + return obj, nil +} + +// +kubebuilder:webhook:path=/mutate-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create,versions=v1,name=mpod.application-networking.k8s.aws,sideEffects=None,webhookVersions=v1,admissionReviewVersions=v1 + +func (m *podMutator) SetupWithManager(mgr ctrl.Manager) { + mgr.GetWebhookServer().Register(apiPathMutatePod, core.MutatingWebhookForMutator(m.scheme, m)) +} diff --git a/pkg/webhook/pod_mutator_test.go b/pkg/webhook/pod_mutator_test.go new file mode 100644 index 00000000..1c899f98 --- /dev/null +++ b/pkg/webhook/pod_mutator_test.go @@ -0,0 +1,81 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestReadinessGateInjectionNew(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + pod := &corev1.Pod{} + + ret, err := m.MutateCreate(context.TODO(), pod) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 1, len(newPod.Spec.ReadinessGates)) + ct := newPod.Spec.ReadinessGates[0].ConditionType + assert.Equal(t, PodReadinessGateConditionType, string(ct)) +} + +func TestReadinessGateAlreadyExists(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + pod := &corev1.Pod{} + prg := corev1.PodReadinessGate{ConditionType: PodReadinessGateConditionType} + pod.Spec.ReadinessGates = append(pod.Spec.ReadinessGates, prg) + + ret, err := m.MutateCreate(context.TODO(), pod) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 1, len(newPod.Spec.ReadinessGates)) + ct := newPod.Spec.ReadinessGates[0].ConditionType + assert.Equal(t, PodReadinessGateConditionType, string(ct)) +} + +func TestUpdateDoesNothing(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + p1 := &corev1.Pod{} + p1.Spec.Hostname = "foo" + p2 := &corev1.Pod{} + p2.Spec.Hostname = "bar" + + ret, err := m.MutateUpdate(context.TODO(), p1, p2) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 0, len(newPod.Spec.ReadinessGates)) +} diff --git a/pkg/webhook/pod_readiness_gate_injector.go b/pkg/webhook/pod_readiness_gate_injector.go new file mode 100644 index 00000000..e2d63bde --- /dev/null +++ b/pkg/webhook/pod_readiness_gate_injector.go @@ -0,0 +1,43 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + PodReadinessGateConditionType = "aws-application-networking-k8s/pod-readiness-gate" +) + +func NewPodReadinessGateInjector(k8sClient client.Client, log gwlog.Logger) *PodReadinessGateInjector { + return &PodReadinessGateInjector{ + k8sClient: k8sClient, + log: log, + } +} + +type PodReadinessGateInjector struct { + k8sClient client.Client + log gwlog.Logger +} + +func (m *PodReadinessGateInjector) Mutate(ctx context.Context, pod *corev1.Pod) error { + pct := corev1.PodConditionType(PodReadinessGateConditionType) + m.log.Debugf("Webhook invoked for pod %s/%s", pod.Name, pod.Namespace) + + found := false + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + found = true + break + } + } + if !found { + pod.Spec.ReadinessGates = append(pod.Spec.ReadinessGates, corev1.PodReadinessGate{ + ConditionType: pct, + }) + } + return nil +} diff --git a/test/suites/webhook/readiness_gate_inject_test.go b/test/suites/webhook/readiness_gate_inject_test.go new file mode 100644 index 00000000..4d030322 --- /dev/null +++ b/test/suites/webhook/readiness_gate_inject_test.go @@ -0,0 +1,85 @@ +package webhook + +import ( + "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/webhook" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("Readiness Gate Inject", Ordered, func() { + untaggedNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-e2e-test-no-tag", + }, + } + taggedNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-e2e-test-tagged", + Labels: map[string]string{ + "aws-application-networking-k8s/pod-readiness-gate-inject": "enabled", + }, + }, + } + + BeforeAll(func() { + err1 := testFramework.Client.Create(ctx, untaggedNS) + err2 := testFramework.Client.Create(ctx, taggedNS) + if err1 != nil || err2 != nil { + Fail("unable to create test namespaces") + } + }) + + It("create deployment in untagged namespace, no readiness gate", func() { + deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "untagged-test-pod", Namespace: untaggedNS.Name}) + testFramework.ExpectCreated(ctx, deployment) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + Expect(len(pods)).To(BeEquivalentTo(1)) + + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + Fail("Pod readiness gate was injected to unlabeled namespace") + } + } + }) + + It("create deployment in tagged namespace, has readiness gate", func() { + deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "tagged-test-pod", Namespace: taggedNS.Name}) + testFramework.ExpectCreated(ctx, deployment) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + Expect(len(pods)).To(BeEquivalentTo(1)) + + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + + foundCount := 0 + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + foundCount++ + } + } + + if foundCount != 1 { + Fail(fmt.Sprintf("Pod readiness gate was expected on labeled namespace. Found %d times", foundCount)) + } + }) + + AfterAll(func() { + err1 := testFramework.Client.Delete(ctx, untaggedNS) + err2 := testFramework.Client.Delete(ctx, taggedNS) + if err1 != nil || err2 != nil { + testFramework.Log.Warn("unable to delete test namespaces") + } + }) +}) diff --git a/test/suites/webhook/suite_test.go b/test/suites/webhook/suite_test.go new file mode 100644 index 00000000..1e9cdadd --- /dev/null +++ b/test/suites/webhook/suite_test.go @@ -0,0 +1,40 @@ +package webhook + +import ( + "context" + "os" + + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/zap" + + "testing" +) + +const ( + k8snamespace = "webhook-" + test.K8sNamespace +) + +var testFramework *test.Framework +var ctx context.Context + +var _ = SynchronizedBeforeSuite(func() { + vpcId := os.Getenv("CLUSTER_VPC_ID") + if vpcId == "" { + Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") + } +}, func() { +}) + +func TestIntegration(t *testing.T) { + ctx = test.NewContext(t) + logger := gwlog.NewLogger(zap.DebugLevel) + testFramework = test.NewFramework(ctx, logger, k8snamespace) + RegisterFailHandler(Fail) + RunSpecs(t, "WebhookIntegration") +} + +var _ = SynchronizedAfterSuite(func() {}, func() { +}) From 8982db2522f1d1164998d1114fc93111bc5b9311 Mon Sep 17 00:00:00 2001 From: erikfu Date: Mon, 26 Feb 2024 16:49:11 -0800 Subject: [PATCH 03/13] point ECR back to public image --- config/manager/kustomization.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2cd86eed..9b968479 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,4 +12,5 @@ configMapGenerator: name: manager-config images: - name: controller - newName: 671123684841.dkr.ecr.us-west-2.amazonaws.com/controller + newName: public.ecr.aws/aws-application-networking-k8s/aws-gateway-controller + newTag: v1.0.3 From 5640db211064922bce6f5e108997485b518e48ae Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 09:55:47 -0800 Subject: [PATCH 04/13] webhook integration test stability improvements --- .../webhook/readiness_gate_inject_test.go | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/test/suites/webhook/readiness_gate_inject_test.go b/test/suites/webhook/readiness_gate_inject_test.go index 4d030322..262b3f39 100644 --- a/test/suites/webhook/readiness_gate_inject_test.go +++ b/test/suites/webhook/readiness_gate_inject_test.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "time" ) var _ = Describe("Readiness Gate Inject", Ordered, func() { @@ -27,59 +28,56 @@ var _ = Describe("Readiness Gate Inject", Ordered, func() { } BeforeAll(func() { - err1 := testFramework.Client.Create(ctx, untaggedNS) - err2 := testFramework.Client.Create(ctx, taggedNS) - if err1 != nil || err2 != nil { - Fail("unable to create test namespaces") - } + testFramework.ExpectDeletedThenNotFound(ctx, untaggedNS, taggedNS) + testFramework.ExpectCreated(ctx, untaggedNS, taggedNS) }) It("create deployment in untagged namespace, no readiness gate", func() { deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "untagged-test-pod", Namespace: untaggedNS.Name}) - testFramework.ExpectCreated(ctx, deployment) - testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + Eventually(func(g Gomega) { + testFramework.ExpectCreated(ctx, deployment) - pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) - Expect(len(pods)).To(BeEquivalentTo(1)) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) - pod := pods[0] - pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + g.Expect(len(pods)).To(BeEquivalentTo(1)) - for _, rg := range pod.Spec.ReadinessGates { - if rg.ConditionType == pct { - Fail("Pod readiness gate was injected to unlabeled namespace") + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + g.Expect(true).To(BeFalse(), "Pod readiness gate was injected to unlabeled namespace") + } } - } + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) }) It("create deployment in tagged namespace, has readiness gate", func() { deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "tagged-test-pod", Namespace: taggedNS.Name}) - testFramework.ExpectCreated(ctx, deployment) - testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + Eventually(func(g Gomega) { + testFramework.ExpectCreated(ctx, deployment) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) - pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) - Expect(len(pods)).To(BeEquivalentTo(1)) + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + g.Expect(len(pods)).To(BeEquivalentTo(1)) - pod := pods[0] - pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) - foundCount := 0 - for _, rg := range pod.Spec.ReadinessGates { - if rg.ConditionType == pct { - foundCount++ + foundCount := 0 + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + foundCount++ + } } - } - if foundCount != 1 { - Fail(fmt.Sprintf("Pod readiness gate was expected on labeled namespace. Found %d times", foundCount)) - } + g.Expect(foundCount).To(Equal(1), + fmt.Sprintf("Pod readiness gate was expected on labeled namespace. Found %d times", foundCount)) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) }) AfterAll(func() { - err1 := testFramework.Client.Delete(ctx, untaggedNS) - err2 := testFramework.Client.Delete(ctx, taggedNS) - if err1 != nil || err2 != nil { - testFramework.Log.Warn("unable to delete test namespaces") - } + testFramework.ExpectDeletedThenNotFound(ctx, untaggedNS, taggedNS) }) }) From dda23bf91a93dcb1e0a88250913eda525969b7cf Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 12:09:36 -0800 Subject: [PATCH 05/13] Additional clarifications for pod readiness gates doc --- docs/guides/pod-readiness-gates.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md index 76130336..5f37f520 100644 --- a/docs/guides/pod-readiness-gates.md +++ b/docs/guides/pod-readiness-gates.md @@ -20,9 +20,23 @@ This prevents the rolling update of a deployment from terminating old pods until ## Setup Pod readiness gates rely on [»admission webhooks«](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), where the Kubernetes API server makes calls to the AWS Gateway API controller as part of pod creation. This call is made using TLS, so the controller must present a TLS certificate. This certificate is stored as a standard Kubernetes secret. If you are using Helm, the certificate will automatically be configured as part of the Helm install. -If you are manually deploying the controller using the ```deploy.yaml``` file, you will need to also create the tls secret in the controller namespace. +If you are manually deploying the controller, for example using the ```deploy.yaml``` file, you will need to also create the tls secret for the webhook in the controller namespace. +### Webhook secret requirements +The webhook requires a specific kubernetes secret to exist in the same namespace as the webhook itself: +* secret name: ```webhook-cert``` +* default controller namespace: ```aws-application-networking-system``` ```console +# example create-secret command, assumes tls.cert and tls.key exist in current directory +kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.cert --key=tls.key +``` + +### Webhook secret configuration example +The below example creates an unsigned certificate, adds it as the webhook secret, then patches the webhook configuration so the API server trusts the certificate. + +If your cluster uses its own PKI and includes appropriate trust configuration for the API server, the certificate issued would be signed by your internal certificate authority and therefore not require the ```kubectl patch``` command below. +```console +# Example commands to configure the webhook to use an unsigned certificate CERT_FILE=tls.crt KEY_FILE=tls.key @@ -42,7 +56,8 @@ openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -keyout ${KEY_FILE} -out $ kubectl create secret tls $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE --cert=${CERT_FILE} --key=${KEY_FILE} # Step 3: after applying deploy.yaml, patch the webhook CA bundle to exactly the cert being used -# this will ensure Kubernetes API server is able to trust the certificate presented by the webhook +# this will ensure Kubernetes API server is able to trust the certificate presented by the webhook. +# This step would not be required if you are using a signed certificate that is trusted by the API server CERT_B64=$(cat tls.crt | base64) kubectl patch mutatingwebhookconfigurations.admissionregistration.k8s.io $WEBHOOK_NAME \ --namespace $WEBHOOK_NAMESPACE --type='json' \ From 25472471579c00a706699435c130e38140a780a4 Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 13:44:54 -0800 Subject: [PATCH 06/13] Add placeholder tls secret so controller can start successfully --- config/manager/manager.yaml | 14 +++++++++++++- docs/guides/pod-readiness-gates.md | 13 +++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 0928f194..4dcf0c5d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -70,4 +70,16 @@ spec: - name: webhook-cert secret: defaultMode: 420 - secretName: webhook-cert \ No newline at end of file + secretName: webhook-cert +--- +# placeholder secret so volume can mount successfully and controller can start +# will not pass validations (no CA, expired, and wrong DNS names) +apiVersion: v1 +kind: Secret +metadata: + name: webhook-cert + namespace: aws-application-networking-system +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURNVENDQWhtZ0F3SUJBZ0lKQUwzOVVONXpjMm9aTUEwR0NTcUdTSWIzRFFFQkN3VUFNQzh4RmpBVUJnTlYKQkFNTURXNXZkQzFoTFhKbFlXd3RZMjR4RlRBVEJnTlZCQW9NREc1dmRDMWhMWEpsWVd3dGJ6QWVGdzB5TkRBeQpNamN5TVRJM01ESmFGdzB5TXpBeU1qY3lNVEkzTURKYU1DOHhGakFVQmdOVkJBTU1EVzV2ZEMxaExYSmxZV3d0ClkyNHhGVEFUQmdOVkJBb01ERzV2ZEMxaExYSmxZV3d0YnpDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVAKQURDQ0FRb0NnZ0VCQU5jd1htN2RtL1NGdDVzY1JFR3NnUzAxN1dDMnErSXQwVWZwRDgvU1UrMmo2YTgyWjNVcgp1Zmg0MGFZNnhGUVZJVlFqa3F2RUNOeVhYU28xaHFFZXhUUWtZRFBGdHE2a2w3cC9TbloyaldkY0NBRHVWR0k3CktLbnkyWWtIdkhkNmg3MVJwNTREZmtDUERvUlZCdU9LU2RIdGtBakptVEVhWkJaWmJIajZHSzhEMnRnYXV2dTMKR0kvNWtMRVZ2clpTRlNYSkZ1aS9ISW5EMk54c29pTllldXB1RHFyZk9KcXBQVmowQUhKVnpTWkZtMmVqcmQvQQoxbE5HdjlBUEl1RkdDakk3bkNVNGNnQkxtQk1zVmpqUmt3RjZtdHlOS1hpSkYzTXVoVHRXNktINm1zcXBCRk1FCmI0NDcveS9YL1RpSjdONHF3UEkwTlVPTjRIcjJzb1VNc3hFQ0F3RUFBYU5RTUU0d0hRWURWUjBPQkJZRUZEUXIKclBieXhBL0dFVkM5MDdnd2lBbWlhcnlwTUI4R0ExVWRJd1FZTUJhQUZEUXJyUGJ5eEEvR0VWQzkwN2d3aUFtaQphcnlwTUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBRXQzOXBTQ05vL0ppY053Ck5mUFNmZDRmWGFRUHZFam9hdk9xZG5Jdm55dFplRUlYZGFPZVpYa3dybFhMaC9SMW1aaURtMC90aEN5YVV1WEIKNDF3YU5TYVdqRWJjalNBTFNPaW9JMExDSjZUTmJJTzg1L1ZJNkMvVTl5dzh3dUhqaXpsVXY2VGFpRllKeXU0QgpRMHE5aStsL1RLR1MvcFNJY3BGS0RiYW5NVXlHQytvajZaVlExdWxUNkZUMnlNK2RLRXdyRFd4ZWhPMUJPTWUwCmVTS1phOUMxWHhqa2VNamdwVVM2R3BKamZrVFluMW5HekV0eng0VUp3ZXIzWDhKR2l0MFdlMStjVURiT0xaRXoKWmh0Nm9MRkpWL0ZMSjIvelJoeEZUazBjN0YrMHdwbldEK3F4Q2ZMbzBOaHlCMkIxcC9CKzZiK3NQdXFUdlNvTgpDaXh1cmg4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFhNRjV1M1p2MGhiZWIKSEVSQnJJRXROZTFndHF2aUxkRkg2US9QMGxQdG8rbXZObWQxSzduNGVOR21Pc1JVRlNGVUk1S3J4QWpjbDEwcQpOWWFoSHNVMEpHQXp4YmF1cEplNmYwcDJkbzFuWEFnQTdsUmlPeWlwOHRtSkI3eDNlb2U5VWFlZUEzNUFqdzZFClZRYmppa25SN1pBSXlaa3hHbVFXV1d4NCtoaXZBOXJZR3JyN3R4aVArWkN4RmI2MlVoVWx5UmJvdnh5Snc5amMKYktJaldIcnFiZzZxM3ppYXFUMVk5QUJ5VmMwbVJadG5vNjNmd05aVFJyL1FEeUxoUmdveU81d2xPSElBUzVnVApMRlk0MFpNQmVwcmNqU2w0aVJkekxvVTdWdWloK3ByS3FRUlRCRytPTy84djEvMDRpZXplS3NEeU5EVkRqZUI2CjlyS0ZETE1SQWdNQkFBRUNnZ0VBT2JHakwvQkVsdnVlN1h4WHBJLytsa09HSUU3NXFJdUdOOVI1dzh0dGF5SnUKVGVhMU9Fbi84MmxaTkVzL1JoZmdOckhPNmpTRjk3YXhhTmF2QU5YQ0k2ZTVEMGhzSVVqSjBWdTllQ055NkFwWgpydjQzSzVzVzNQSGFkdzNXN3VXd0xRY09mS1FOSG52OGRXaGlqM0VOTjdhTXpuNVdqejYzSlBMV3pWeW9iNHVJCkxKd3pCWkR3aHJyQjdMTXFKSzlpdUZiaWF2NkpBVFJGVjdBSHU4bnFRSWxBbTkrQnlxdXZQZkhPbXNtQUNsbHcKeTdOWnhLMFdSZmJPUzd4TGFFd1FHa3NEZXFRN2NQYUNOakkxWnArbUlLZEVpQ1VpQXZPdkNoSHd0TjNYNlNGbgpPcXNaQ3hvaE55L01Nam5mdWhpeXVvVi9WSlFLYmNHMk9TZzlGd0VMd1FLQmdRRHRnL0FETFRpWnUzNjFsckJrCkh0bVU0VUFXRlJVWGhnTmNvU0hyOTdRVWJ4TURTVEUvVjdYN0E1aFcvYzNiWC9Fcnd3bTBZeGZHY2FzNHJSSFQKWE9na3pYaDMxU01HWXlpVkF1cEtJTzJpQ2MxN1UxYkFLbEVOSDFtbnY1UE1PYkZDQm92S25leXdpUHJiSXZEaQp6SE5jOE9IL2l0R0dncDFCL3h1NlpucGplUUtCZ1FEbjc1NFpVc3pLVlhIVDdhTWhjOU43UHNrMjRadVlpTVJlCkMxYWIxTjhoRkxpVUNwRXN1NXVKcEV0d0tNL2dXaEpjaWxTaHRKK3gwZkczanNRSk5mWXZVRmxJa1JGVFBIdlQKS2RYNkVzYWFIVkZuM2dweitCcHZjVy9idk9pbEtCNDNSUlFKU2tnUmhkUnEvUjk4MnBzYWFBc09WRm96L28xWAowdWF4ckN1T1dRS0JnRzljMVVRb0I4bk01M1FzMnplV1gxNDIzdDE3dFEvNmZja0lvK2NIbFIrZmxNS05wdEdVClJuY1RFSEo1UGZRRjRBWXN4SGdYbmlZbFZhcVZPeTVtK1ZHSUpWdktTMG5MWkZPNXNqQmZrQXZSbk02ZUhLYXQKTUtOK2Q0TDNpRXpSSUJOZERsNUovWmdvSWJadGc1UlRXQ1BUcmFNcmEySXVDNTNPQnlvMnNsdkJBb0dBTkYrdwpsTWxVdzAvZUgxd25IVE81aXJnWDJkUENQZ1NNU3l4R1IzUWZXcW9DTURQZXFucEcyaU1HZ2ZKRlZzVWdKbE42Clh2V1pwaDdoZFhEQXBjL1Fvc2lERU5icFVhRnoyTEEyeUh5YTZrdzZpTGprSldIZUhsSkFUeDl1YlhVTXRiQmMKb09oc004RER3ZEVjM2lYREpvaGVEc05QaHpReEdLYmdQempBc1NrQ2dZRUFoNk1UWHFHeGlLbDU3WTFOb1lYTwo3U0EzRXF4R1BjL091NVh4RWdrTnRRV0hnMWN1cmtBeU1UaXRrMklzRVhCL2RISjN6Y2hiQjlEcThCRDBBeDl2CkM0QUdMVWdiOTZhREl6dmwwbXk4MHZ6dkZGaEt3Z3UzeXFTaFhQclZNSS8wM045ckx6VUJlaWxoRHE5UFVjR24KcTJTeTNUTS9CenlaYlgyQjdYK0gyVms9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K \ No newline at end of file diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md index 5f37f520..e0f5d44d 100644 --- a/docs/guides/pod-readiness-gates.md +++ b/docs/guides/pod-readiness-gates.md @@ -20,7 +20,7 @@ This prevents the rolling update of a deployment from terminating old pods until ## Setup Pod readiness gates rely on [»admission webhooks«](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), where the Kubernetes API server makes calls to the AWS Gateway API controller as part of pod creation. This call is made using TLS, so the controller must present a TLS certificate. This certificate is stored as a standard Kubernetes secret. If you are using Helm, the certificate will automatically be configured as part of the Helm install. -If you are manually deploying the controller, for example using the ```deploy.yaml``` file, you will need to also create the tls secret for the webhook in the controller namespace. +If you are manually deploying the controller, for example using the ```deploy.yaml``` file, you will need to create the tls secret for the webhook in the controller namespace. The ```deploy.yaml``` file includes a placeholder secret, but it must be updated if you wish to use the webhook. The placeholder secret _will not_ pass API server validations, but will ensure the controller container is able to start. ### Webhook secret requirements The webhook requires a specific kubernetes secret to exist in the same namespace as the webhook itself: @@ -28,6 +28,7 @@ The webhook requires a specific kubernetes secret to exist in the same namespace * default controller namespace: ```aws-application-networking-system``` ```console # example create-secret command, assumes tls.cert and tls.key exist in current directory +# if the placeholder secret exists, you will need to delete it before setting the new value kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.cert --key=tls.key ``` @@ -51,13 +52,13 @@ HOST=${WEBHOOK_SVC_NAME}.${WEBHOOK_NAMESPACE}.svc openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" \ -addext "subjectAltName = DNS:${HOST}, DNS:${HOST}.cluster.local" -# Step 2: add the secret - can be done so long as the namespace exists -# note that running "kubectl delete -f deploy.yaml" will remove the controller namespace AND this secret. +# Step 2: replace the placeholder secret from deploy.yaml +kubectl delete secret $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE kubectl create secret tls $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE --cert=${CERT_FILE} --key=${KEY_FILE} -# Step 3: after applying deploy.yaml, patch the webhook CA bundle to exactly the cert being used -# this will ensure Kubernetes API server is able to trust the certificate presented by the webhook. -# This step would not be required if you are using a signed certificate that is trusted by the API server +# Step 3: Patch the webhook CA bundle to exactly the cert being used. +# This will ensure Kubernetes API server is able to trust the certificate presented by the webhook. +# This step would not be required if you are using a signed certificate that is already trusted by the API server CERT_B64=$(cat tls.crt | base64) kubectl patch mutatingwebhookconfigurations.admissionregistration.k8s.io $WEBHOOK_NAME \ --namespace $WEBHOOK_NAMESPACE --type='json' \ From 30922a1b09104f229fefbf80e54c08ba877ae4ce Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 14:34:36 -0800 Subject: [PATCH 07/13] removed placeholder ca value for webhook --- config/webhook/manifests.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 2b5432b8..8ccbe5f9 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -8,9 +8,6 @@ webhooks: - admissionReviewVersions: - v1 clientConfig: - # placeholder newline for caBundle, can replace with actual CA before install using yq - # or afterward with a patch on the webhook - caBundle: Cg== service: name: webhook-service namespace: aws-application-networking-system From ea7e3033c86ff6fc469c5b8dded8f1e068fd376b Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 14:40:27 -0800 Subject: [PATCH 08/13] webhook integration test stability fixes --- .../suites/webhook/readiness_gate_inject_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/suites/webhook/readiness_gate_inject_test.go b/test/suites/webhook/readiness_gate_inject_test.go index 262b3f39..38d41e66 100644 --- a/test/suites/webhook/readiness_gate_inject_test.go +++ b/test/suites/webhook/readiness_gate_inject_test.go @@ -28,15 +28,21 @@ var _ = Describe("Readiness Gate Inject", Ordered, func() { } BeforeAll(func() { - testFramework.ExpectDeletedThenNotFound(ctx, untaggedNS, taggedNS) - testFramework.ExpectCreated(ctx, untaggedNS, taggedNS) + Eventually(func(g Gomega) { + _ = testFramework.Delete(ctx, untaggedNS) + _ = testFramework.Delete(ctx, taggedNS) + testFramework.EventuallyExpectNotFound(ctx, untaggedNS, taggedNS) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + + Eventually(func(g Gomega) { + testFramework.ExpectCreated(ctx, untaggedNS, taggedNS) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) }) It("create deployment in untagged namespace, no readiness gate", func() { deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "untagged-test-pod", Namespace: untaggedNS.Name}) Eventually(func(g Gomega) { - testFramework.ExpectCreated(ctx, deployment) - + testFramework.Create(ctx, deployment) testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) @@ -56,7 +62,7 @@ var _ = Describe("Readiness Gate Inject", Ordered, func() { It("create deployment in tagged namespace, has readiness gate", func() { deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "tagged-test-pod", Namespace: taggedNS.Name}) Eventually(func(g Gomega) { - testFramework.ExpectCreated(ctx, deployment) + testFramework.Create(ctx, deployment) testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) From 33171e32ea3cd926f37dde440652da0ac854eab6 Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 14:59:05 -0800 Subject: [PATCH 09/13] remove kubebuilder directive for webhook - not used --- pkg/webhook/pod_mutator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/webhook/pod_mutator.go b/pkg/webhook/pod_mutator.go index 99826f47..e3f13346 100644 --- a/pkg/webhook/pod_mutator.go +++ b/pkg/webhook/pod_mutator.go @@ -43,8 +43,6 @@ func (m *podMutator) MutateUpdate(ctx context.Context, obj runtime.Object, oldOb return obj, nil } -// +kubebuilder:webhook:path=/mutate-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create,versions=v1,name=mpod.application-networking.k8s.aws,sideEffects=None,webhookVersions=v1,admissionReviewVersions=v1 - func (m *podMutator) SetupWithManager(mgr ctrl.Manager) { mgr.GetWebhookServer().Register(apiPathMutatePod, core.MutatingWebhookForMutator(m.scheme, m)) } From cc2f234cb40c79d4374445e3e30f9bd66057a019 Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 16:33:55 -0800 Subject: [PATCH 10/13] Shift placeholder cert generation for deploy.yaml to make-deploy --- .gitignore | 6 +++++- Makefile | 6 ++++++ config/manager/manager.yaml | 6 +++--- docs/guides/pod-readiness-gates.md | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index a0534045..b9560ab7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,8 @@ go.work* pkg/aws/services/gomock_reflect_* # Image build tarballed bundles -*.tgz \ No newline at end of file +*.tgz + +# generated during make-deploy +tls.crt +tls.key \ No newline at end of file diff --git a/Makefile b/Makefile index 211fdcfc..614adfb2 100644 --- a/Makefile +++ b/Makefile @@ -98,10 +98,16 @@ docker-build: test ## Build docker image with the manager. docker-push: ## Push docker image with the manager. docker push ${IMG} +# also generates a placeholder cert for the webhook - this cert is not intended to be valid .PHONY: build-deploy build-deploy: ## Create a deployment file that can be applied with `kubectl apply -f deploy.yaml` cd config/manager && kustomize edit set image controller=${ECRIMAGES} kustomize build config/default > deploy.yaml + openssl req -x509 -nodes -days 1 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=not-a-real-cn/O=not-a-real-o" > /dev/null 2>&1 + export KEY_B64=`cat tls.key | base64` + export CERT_B64=`cat tls.crt | base64` + yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.crt") = env(CERT_B64)' deploy.yaml 2>&1 + yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.key") = env(KEY_B64)' deploy.yaml 2>&1 .PHONY: manifest manifest: ## Generate CRD manifest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4dcf0c5d..54b71ded 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -73,7 +73,7 @@ spec: secretName: webhook-cert --- # placeholder secret so volume can mount successfully and controller can start -# will not pass validations (no CA, expired, and wrong DNS names) +# populated during make-deploy. Will not pass validations (no CA, expires after 1 day, wrong DNS names) apiVersion: v1 kind: Secret metadata: @@ -81,5 +81,5 @@ metadata: namespace: aws-application-networking-system type: kubernetes.io/tls data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURNVENDQWhtZ0F3SUJBZ0lKQUwzOVVONXpjMm9aTUEwR0NTcUdTSWIzRFFFQkN3VUFNQzh4RmpBVUJnTlYKQkFNTURXNXZkQzFoTFhKbFlXd3RZMjR4RlRBVEJnTlZCQW9NREc1dmRDMWhMWEpsWVd3dGJ6QWVGdzB5TkRBeQpNamN5TVRJM01ESmFGdzB5TXpBeU1qY3lNVEkzTURKYU1DOHhGakFVQmdOVkJBTU1EVzV2ZEMxaExYSmxZV3d0ClkyNHhGVEFUQmdOVkJBb01ERzV2ZEMxaExYSmxZV3d0YnpDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVAKQURDQ0FRb0NnZ0VCQU5jd1htN2RtL1NGdDVzY1JFR3NnUzAxN1dDMnErSXQwVWZwRDgvU1UrMmo2YTgyWjNVcgp1Zmg0MGFZNnhGUVZJVlFqa3F2RUNOeVhYU28xaHFFZXhUUWtZRFBGdHE2a2w3cC9TbloyaldkY0NBRHVWR0k3CktLbnkyWWtIdkhkNmg3MVJwNTREZmtDUERvUlZCdU9LU2RIdGtBakptVEVhWkJaWmJIajZHSzhEMnRnYXV2dTMKR0kvNWtMRVZ2clpTRlNYSkZ1aS9ISW5EMk54c29pTllldXB1RHFyZk9KcXBQVmowQUhKVnpTWkZtMmVqcmQvQQoxbE5HdjlBUEl1RkdDakk3bkNVNGNnQkxtQk1zVmpqUmt3RjZtdHlOS1hpSkYzTXVoVHRXNktINm1zcXBCRk1FCmI0NDcveS9YL1RpSjdONHF3UEkwTlVPTjRIcjJzb1VNc3hFQ0F3RUFBYU5RTUU0d0hRWURWUjBPQkJZRUZEUXIKclBieXhBL0dFVkM5MDdnd2lBbWlhcnlwTUI4R0ExVWRJd1FZTUJhQUZEUXJyUGJ5eEEvR0VWQzkwN2d3aUFtaQphcnlwTUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBRXQzOXBTQ05vL0ppY053Ck5mUFNmZDRmWGFRUHZFam9hdk9xZG5Jdm55dFplRUlYZGFPZVpYa3dybFhMaC9SMW1aaURtMC90aEN5YVV1WEIKNDF3YU5TYVdqRWJjalNBTFNPaW9JMExDSjZUTmJJTzg1L1ZJNkMvVTl5dzh3dUhqaXpsVXY2VGFpRllKeXU0QgpRMHE5aStsL1RLR1MvcFNJY3BGS0RiYW5NVXlHQytvajZaVlExdWxUNkZUMnlNK2RLRXdyRFd4ZWhPMUJPTWUwCmVTS1phOUMxWHhqa2VNamdwVVM2R3BKamZrVFluMW5HekV0eng0VUp3ZXIzWDhKR2l0MFdlMStjVURiT0xaRXoKWmh0Nm9MRkpWL0ZMSjIvelJoeEZUazBjN0YrMHdwbldEK3F4Q2ZMbzBOaHlCMkIxcC9CKzZiK3NQdXFUdlNvTgpDaXh1cmg4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFhNRjV1M1p2MGhiZWIKSEVSQnJJRXROZTFndHF2aUxkRkg2US9QMGxQdG8rbXZObWQxSzduNGVOR21Pc1JVRlNGVUk1S3J4QWpjbDEwcQpOWWFoSHNVMEpHQXp4YmF1cEplNmYwcDJkbzFuWEFnQTdsUmlPeWlwOHRtSkI3eDNlb2U5VWFlZUEzNUFqdzZFClZRYmppa25SN1pBSXlaa3hHbVFXV1d4NCtoaXZBOXJZR3JyN3R4aVArWkN4RmI2MlVoVWx5UmJvdnh5Snc5amMKYktJaldIcnFiZzZxM3ppYXFUMVk5QUJ5VmMwbVJadG5vNjNmd05aVFJyL1FEeUxoUmdveU81d2xPSElBUzVnVApMRlk0MFpNQmVwcmNqU2w0aVJkekxvVTdWdWloK3ByS3FRUlRCRytPTy84djEvMDRpZXplS3NEeU5EVkRqZUI2CjlyS0ZETE1SQWdNQkFBRUNnZ0VBT2JHakwvQkVsdnVlN1h4WHBJLytsa09HSUU3NXFJdUdOOVI1dzh0dGF5SnUKVGVhMU9Fbi84MmxaTkVzL1JoZmdOckhPNmpTRjk3YXhhTmF2QU5YQ0k2ZTVEMGhzSVVqSjBWdTllQ055NkFwWgpydjQzSzVzVzNQSGFkdzNXN3VXd0xRY09mS1FOSG52OGRXaGlqM0VOTjdhTXpuNVdqejYzSlBMV3pWeW9iNHVJCkxKd3pCWkR3aHJyQjdMTXFKSzlpdUZiaWF2NkpBVFJGVjdBSHU4bnFRSWxBbTkrQnlxdXZQZkhPbXNtQUNsbHcKeTdOWnhLMFdSZmJPUzd4TGFFd1FHa3NEZXFRN2NQYUNOakkxWnArbUlLZEVpQ1VpQXZPdkNoSHd0TjNYNlNGbgpPcXNaQ3hvaE55L01Nam5mdWhpeXVvVi9WSlFLYmNHMk9TZzlGd0VMd1FLQmdRRHRnL0FETFRpWnUzNjFsckJrCkh0bVU0VUFXRlJVWGhnTmNvU0hyOTdRVWJ4TURTVEUvVjdYN0E1aFcvYzNiWC9Fcnd3bTBZeGZHY2FzNHJSSFQKWE9na3pYaDMxU01HWXlpVkF1cEtJTzJpQ2MxN1UxYkFLbEVOSDFtbnY1UE1PYkZDQm92S25leXdpUHJiSXZEaQp6SE5jOE9IL2l0R0dncDFCL3h1NlpucGplUUtCZ1FEbjc1NFpVc3pLVlhIVDdhTWhjOU43UHNrMjRadVlpTVJlCkMxYWIxTjhoRkxpVUNwRXN1NXVKcEV0d0tNL2dXaEpjaWxTaHRKK3gwZkczanNRSk5mWXZVRmxJa1JGVFBIdlQKS2RYNkVzYWFIVkZuM2dweitCcHZjVy9idk9pbEtCNDNSUlFKU2tnUmhkUnEvUjk4MnBzYWFBc09WRm96L28xWAowdWF4ckN1T1dRS0JnRzljMVVRb0I4bk01M1FzMnplV1gxNDIzdDE3dFEvNmZja0lvK2NIbFIrZmxNS05wdEdVClJuY1RFSEo1UGZRRjRBWXN4SGdYbmlZbFZhcVZPeTVtK1ZHSUpWdktTMG5MWkZPNXNqQmZrQXZSbk02ZUhLYXQKTUtOK2Q0TDNpRXpSSUJOZERsNUovWmdvSWJadGc1UlRXQ1BUcmFNcmEySXVDNTNPQnlvMnNsdkJBb0dBTkYrdwpsTWxVdzAvZUgxd25IVE81aXJnWDJkUENQZ1NNU3l4R1IzUWZXcW9DTURQZXFucEcyaU1HZ2ZKRlZzVWdKbE42Clh2V1pwaDdoZFhEQXBjL1Fvc2lERU5icFVhRnoyTEEyeUh5YTZrdzZpTGprSldIZUhsSkFUeDl1YlhVTXRiQmMKb09oc004RER3ZEVjM2lYREpvaGVEc05QaHpReEdLYmdQempBc1NrQ2dZRUFoNk1UWHFHeGlLbDU3WTFOb1lYTwo3U0EzRXF4R1BjL091NVh4RWdrTnRRV0hnMWN1cmtBeU1UaXRrMklzRVhCL2RISjN6Y2hiQjlEcThCRDBBeDl2CkM0QUdMVWdiOTZhREl6dmwwbXk4MHZ6dkZGaEt3Z3UzeXFTaFhQclZNSS8wM045ckx6VUJlaWxoRHE5UFVjR24KcTJTeTNUTS9CenlaYlgyQjdYK0gyVms9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K \ No newline at end of file + tls.crt: Cg== + tls.key: Cg== \ No newline at end of file diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md index e0f5d44d..8c62906f 100644 --- a/docs/guides/pod-readiness-gates.md +++ b/docs/guides/pod-readiness-gates.md @@ -27,9 +27,9 @@ The webhook requires a specific kubernetes secret to exist in the same namespace * secret name: ```webhook-cert``` * default controller namespace: ```aws-application-networking-system``` ```console -# example create-secret command, assumes tls.cert and tls.key exist in current directory +# example create-secret command, assumes tls.crt and tls.key exist in current directory # if the placeholder secret exists, you will need to delete it before setting the new value -kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.cert --key=tls.key +kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.crt --key=tls.key ``` ### Webhook secret configuration example From 02831a892a771021034a57eccef631f2e1974c8f Mon Sep 17 00:00:00 2001 From: erikfu Date: Tue, 27 Feb 2024 16:45:12 -0800 Subject: [PATCH 11/13] Small fix for cert env variables in Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 614adfb2..3956c0d3 100644 --- a/Makefile +++ b/Makefile @@ -104,8 +104,8 @@ build-deploy: ## Create a deployment file that can be applied with `kubectl appl cd config/manager && kustomize edit set image controller=${ECRIMAGES} kustomize build config/default > deploy.yaml openssl req -x509 -nodes -days 1 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=not-a-real-cn/O=not-a-real-o" > /dev/null 2>&1 - export KEY_B64=`cat tls.key | base64` - export CERT_B64=`cat tls.crt | base64` + $(eval export KEY_B64 := $(shell cat tls.key | base64)) + $(eval export CERT_B64 := $(shell cat tls.crt | base64)) yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.crt") = env(CERT_B64)' deploy.yaml 2>&1 yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.key") = env(KEY_B64)' deploy.yaml 2>&1 From 9e19b6ab8a35faacde53837be089173c29bbfc28 Mon Sep 17 00:00:00 2001 From: erikfu Date: Thu, 29 Feb 2024 13:58:00 -0800 Subject: [PATCH 12/13] Updates from PR feedback --- .gitignore | 6 +----- Makefile | 1 + docs/guides/pod-readiness-gates.md | 12 +++--------- pkg/webhook/pod_readiness_gate_injector.go | 2 +- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index b9560ab7..a0534045 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,4 @@ go.work* pkg/aws/services/gomock_reflect_* # Image build tarballed bundles -*.tgz - -# generated during make-deploy -tls.crt -tls.key \ No newline at end of file +*.tgz \ No newline at end of file diff --git a/Makefile b/Makefile index 3956c0d3..6aca480f 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ build-deploy: ## Create a deployment file that can be applied with `kubectl appl $(eval export CERT_B64 := $(shell cat tls.crt | base64)) yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.crt") = env(CERT_B64)' deploy.yaml 2>&1 yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.key") = env(KEY_B64)' deploy.yaml 2>&1 + rm tls.key tls.crt .PHONY: manifest manifest: ## Generate CRD manifest diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md index 8c62906f..189269a9 100644 --- a/docs/guides/pod-readiness-gates.md +++ b/docs/guides/pod-readiness-gates.md @@ -1,7 +1,5 @@ # Pod readiness gate -# TODO: Update to reflect final implementation of readiness gate logic. Replace - AWS Gateway API controller supports [»Pod readiness gates«](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate) to indicate that pod is registered to the VPC Lattice and healthy to receive traffic. The controller automatically injects the necessary readiness gate configuration to the pod spec via mutating webhook during pod creation. @@ -85,7 +83,7 @@ Status: Active Once labelled, the controller will add the pod readiness gates to all subsequently created pods. -The readiness gates have the prefix `` and the controller injects the config to the pod spec only during pod creation. +The readiness gates have the condition type ```application-networking.k8s.aws/pod-readiness-gate``` and the controller injects the config to the pod spec only during pod creation. ## Object Selector The default webhook configuration matches all pods in the namespaces containing the label `aws-application-networking-k8s/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select ONLY pods with `aws-application-networking-k8s/pod-readiness-gate-inject: enabled` label instead of all pods in the labeled namespace, you can add the following `objectSelector` to the webhook: @@ -112,9 +110,6 @@ $ kubectl edit mutatingwebhookconfigurations aws-appnet-gwc-mutating-webhook ``` When you specify multiple selectors, pods matching all the conditions will get mutated. -## Disabling the readiness gate inject -You can specify the controller flag `--enable-pod-readiness-gate-inject=false` during controller startup to disable the controller from modifying the pod spec. - ## Checking the pod condition status The status of the readiness gates can be verified with `kubectl get pod -o wide`: @@ -132,13 +127,12 @@ nginx-test-5744b9ff84-7ftl9 1/1 Running 0 81s 10.1.2.3 ip-1 If a readiness gate doesn't get ready, you can check the reason via: ```console -$ kubectl get pod nginx-test-545d8f4d89-l7rcl -o yaml | grep -B7 'type: ' +$ kubectl get pod nginx-test-545d8f4d89-l7rcl -o yaml | grep -B7 'type: application-networking.k8s.aws/pod-readiness-gate' status: conditions: - lastProbeTime: null lastTransitionTime: null - message: reason: HEALTHY status: "True" - type: + type: application-networking.k8s.aws/pod-readiness-gate ``` diff --git a/pkg/webhook/pod_readiness_gate_injector.go b/pkg/webhook/pod_readiness_gate_injector.go index e2d63bde..3615a121 100644 --- a/pkg/webhook/pod_readiness_gate_injector.go +++ b/pkg/webhook/pod_readiness_gate_injector.go @@ -8,7 +8,7 @@ import ( ) const ( - PodReadinessGateConditionType = "aws-application-networking-k8s/pod-readiness-gate" + PodReadinessGateConditionType = "application-networking.k8s.aws/pod-readiness-gate" ) func NewPodReadinessGateInjector(k8sClient client.Client, log gwlog.Logger) *PodReadinessGateInjector { From 3df31c1973514a71574c8f13859e0e15e9ae020c Mon Sep 17 00:00:00 2001 From: erikfu Date: Thu, 29 Feb 2024 14:04:00 -0800 Subject: [PATCH 13/13] changed namespace label for webhook enabling to use 'application-networking.k8s.aws' --- config/webhook/manifests.yaml | 2 +- docs/guides/pod-readiness-gates.md | 14 +++++++------- test/suites/webhook/readiness_gate_inject_test.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 8ccbe5f9..0718d682 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -26,7 +26,7 @@ webhooks: sideEffects: None namespaceSelector: matchExpressions: - - key: aws-application-networking-k8s/pod-readiness-gate-inject + - key: application-networking.k8s.aws/pod-readiness-gate-inject operator: In values: - enabled diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md index 189269a9..4ffb57db 100644 --- a/docs/guides/pod-readiness-gates.md +++ b/docs/guides/pod-readiness-gates.md @@ -3,7 +3,7 @@ AWS Gateway API controller supports [»Pod readiness gates«](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate) to indicate that pod is registered to the VPC Lattice and healthy to receive traffic. The controller automatically injects the necessary readiness gate configuration to the pod spec via mutating webhook during pod creation. -For readiness gate configuration to be injected to the pod spec, you need to apply the label `aws-application-networking-k8s/pod-readiness-gate-inject: enabled` to the pod namespace. +For readiness gate configuration to be injected to the pod spec, you need to apply the label `application-networking.k8s.aws/pod-readiness-gate-inject: enabled` to the pod namespace. The pod readiness gate is needed under certain circumstances to achieve full zero downtime rolling deployments. Consider the following example: @@ -70,12 +70,12 @@ Pod readiness gate support is enabled by default on the AWS Gateway API controll $ kubectl create namespace example-ns namespace/example-ns created -$ kubectl label namespace example-ns aws-application-networking-k8s/pod-readiness-gate-inject=enabled +$ kubectl label namespace example-ns application-networking.k8s.aws/pod-readiness-gate-inject=enabled namespace/example-ns labeled $ kubectl describe namespace example-ns Name: example-ns -Labels: aws-application-networking-k8s/pod-readiness-gate-inject=enabled +Labels: application-networking.k8s.aws/pod-readiness-gate-inject=enabled kubernetes.io/metadata.name=example-ns Annotations: Status: Active @@ -86,11 +86,11 @@ Once labelled, the controller will add the pod readiness gates to all subsequent The readiness gates have the condition type ```application-networking.k8s.aws/pod-readiness-gate``` and the controller injects the config to the pod spec only during pod creation. ## Object Selector -The default webhook configuration matches all pods in the namespaces containing the label `aws-application-networking-k8s/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select ONLY pods with `aws-application-networking-k8s/pod-readiness-gate-inject: enabled` label instead of all pods in the labeled namespace, you can add the following `objectSelector` to the webhook: +The default webhook configuration matches all pods in the namespaces containing the label `application-networking.k8s.aws/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select ONLY pods with `application-networking.k8s.aws/pod-readiness-gate-inject: enabled` label instead of all pods in the labeled namespace, you can add the following `objectSelector` to the webhook: ``` objectSelector: matchLabels: - aws-application-networking-k8s/pod-readiness-gate-inject: enabled + application-networking.k8s.aws/pod-readiness-gate-inject: enabled ``` To edit, ``` @@ -99,13 +99,13 @@ $ kubectl edit mutatingwebhookconfigurations aws-appnet-gwc-mutating-webhook name: mpod.gwc.k8s.aws namespaceSelector: matchExpressions: - - key: aws-application-networking-k8s/pod-readiness-gate-inject + - key: application-networking.k8s.aws/pod-readiness-gate-inject operator: In values: - enabled objectSelector: matchLabels: - aws-application-networking-k8s/pod-readiness-gate-inject: enabled + application-networking.k8s.aws/pod-readiness-gate-inject: enabled ... ``` When you specify multiple selectors, pods matching all the conditions will get mutated. diff --git a/test/suites/webhook/readiness_gate_inject_test.go b/test/suites/webhook/readiness_gate_inject_test.go index 38d41e66..861cefab 100644 --- a/test/suites/webhook/readiness_gate_inject_test.go +++ b/test/suites/webhook/readiness_gate_inject_test.go @@ -22,7 +22,7 @@ var _ = Describe("Readiness Gate Inject", Ordered, func() { ObjectMeta: metav1.ObjectMeta{ Name: "webhook-e2e-test-tagged", Labels: map[string]string{ - "aws-application-networking-k8s/pod-readiness-gate-inject": "enabled", + "application-networking.k8s.aws/pod-readiness-gate-inject": "enabled", }, }, }