diff --git a/Makefile b/Makefile index 0d047a9d..6aca480f 100644 --- a/Makefile +++ b/Makefile @@ -98,10 +98,17 @@ docker-build: test ## Build docker image with the manager. docker-push: ## Push docker image with the manager. docker push ${IMG} +# also generates a placeholder cert for the webhook - this cert is not intended to be valid .PHONY: build-deploy build-deploy: ## Create a deployment file that can be applied with `kubectl apply -f deploy.yaml` cd config/manager && kustomize edit set image controller=${ECRIMAGES} kustomize build config/default > deploy.yaml + openssl req -x509 -nodes -days 1 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=not-a-real-cn/O=not-a-real-o" > /dev/null 2>&1 + $(eval export KEY_B64 := $(shell cat tls.key | base64)) + $(eval export CERT_B64 := $(shell cat tls.crt | base64)) + yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.crt") = env(CERT_B64)' deploy.yaml 2>&1 + yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.key") = env(KEY_B64)' deploy.yaml 2>&1 + rm tls.key tls.crt .PHONY: manifest manifest: ## Generate CRD manifest @@ -144,3 +151,20 @@ api-reference: ## Update documentation in docs/api-reference.md docs: mkdir -p site mkdocs build + +# NB webhook tests can only run if the controller is deployed to the cluster +webhook-e2e-test-namespace := "webhook-e2e-test" + +.PHONY: webhook-e2e-test +webhook-e2e-test: + @kubectl create namespace $(webhook-e2e-test-namespace) > /dev/null 2>&1 || true # ignore already exists error + LOG_LEVEL=debug + cd test && go test \ + -p 1 \ + -count 1 \ + -timeout 10m \ + -v \ + ./suites/webhook/... \ + --ginkgo.focus="${FOCUS}" \ + --ginkgo.skip="${SKIP}" \ + --ginkgo.v \ No newline at end of file diff --git a/cmd/aws-application-networking-k8s/main.go b/cmd/aws-application-networking-k8s/main.go index 6ee85ee3..84848b2c 100644 --- a/cmd/aws-application-networking-k8s/main.go +++ b/cmd/aws-application-networking-k8s/main.go @@ -18,10 +18,11 @@ package main import ( "flag" - "os" - + "github.com/aws/aws-application-networking-k8s/pkg/webhook" "github.com/go-logr/zapr" "go.uber.org/zap/zapcore" + "os" + k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -49,7 +50,6 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/k8s" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) var ( @@ -128,14 +128,24 @@ func main() { setupLog.Fatal("cloud client setup failed: %s", err) } + // do not create the webhook server when running locally + var webhookServer k8swebhook.Server + isLocalDev := config.DevMode != "" + if !isLocalDev { + webhookServer = k8swebhook.NewServer(k8swebhook.Options{ + Port: 9443, + CertDir: "/etc/webhook-cert/", + CertName: "tls.crt", + KeyName: "tls.key", + }) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr, }, - WebhookServer: webhook.NewServer(webhook.Options{ - Port: 9443, - }), + WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "amazon-vpc-lattice.io", @@ -144,6 +154,15 @@ func main() { setupLog.Fatal("manager setup failed:", err) } + if !isLocalDev { + // register webhook handlers + readinessGateInjector := webhook.NewPodReadinessGateInjector( + mgr.GetClient(), + log.Named("pod-readiness-gate-injector"), + ) + webhook.NewPodMutator(scheme, readinessGateInjector).SetupWithManager(mgr) + } + finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient()) // parent logging scope for all controllers diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index b542098c..3acfe15b 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,26 +1,10 @@ -# Adds namespace to all resources. -#namespace: code-aws-application-networking-system - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -#namePrefix: code- - -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue - -bases: +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: - ../crds - ../rbac - ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -33,42 +17,3 @@ patchesStrategicMerge: # Mount the controller config file for loading manager configurations # through a ComponentConfig type #- manager_config_patch.yaml - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c2995bbc..54b71ded 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -60,5 +60,26 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - mountPath: /etc/webhook-cert + name: webhook-cert + readOnly: true serviceAccountName: gateway-api-controller terminationGracePeriodSeconds: 10 + volumes: + - name: webhook-cert + secret: + defaultMode: 420 + secretName: webhook-cert +--- +# placeholder secret so volume can mount successfully and controller can start +# populated during make-deploy. Will not pass validations (no CA, expires after 1 day, wrong DNS names) +apiVersion: v1 +kind: Secret +metadata: + name: webhook-cert + namespace: aws-application-networking-system +type: kubernetes.io/tls +data: + tls.crt: Cg== + tls.key: Cg== \ No newline at end of file diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed137168..7f6a57fb 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - monitor.yaml diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index f0549f47..7f06e49a 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: # All RBAC will be applied under this service account in # the deployment namespace. You may comment out this resource diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..cd08260e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - manifests.yaml \ No newline at end of file diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 00000000..0718d682 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: aws-appnet-gwc-mutating-webhook +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: aws-application-networking-system + path: /mutate-pod + failurePolicy: Fail + name: mpod.gwc.k8s.aws + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + namespaceSelector: + matchExpressions: + - key: application-networking.k8s.aws/pod-readiness-gate-inject + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - gateway-api-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: aws-application-networking-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: gateway-api-controller \ No newline at end of file diff --git a/docs/guides/pod-readiness-gates.md b/docs/guides/pod-readiness-gates.md new file mode 100644 index 00000000..4ffb57db --- /dev/null +++ b/docs/guides/pod-readiness-gates.md @@ -0,0 +1,138 @@ +# Pod readiness gate + +AWS Gateway API controller supports [»Pod readiness gates«](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate) to indicate that pod is registered to the VPC Lattice and healthy to receive traffic. +The controller automatically injects the necessary readiness gate configuration to the pod spec via mutating webhook during pod creation. + +For readiness gate configuration to be injected to the pod spec, you need to apply the label `application-networking.k8s.aws/pod-readiness-gate-inject: enabled` to the pod namespace. + +The pod readiness gate is needed under certain circumstances to achieve full zero downtime rolling deployments. Consider the following example: + +* Low number of replicas in a deployment +* Start a rolling update of the deployment +* Rollout of new pods takes less time than it takes the AWS Gateway API controller to register the new pods and for their health state turn »Healthy« in the target group +* At some point during this rolling update, the target group might only have registered targets that are in »Initial« or »Draining« state; this results in service outage + +In order to avoid this situation, the AWS Gateway API controller can set the readiness condition on the pods that constitute your ingress or service backend. The condition status on a pod will be set to `True` only when the corresponding target in the VPC Lattice target group shows a health state of »Healthy«. +This prevents the rolling update of a deployment from terminating old pods until the newly created pods are »Healthy« in the VPC Lattice target group and ready to take traffic. + +## Setup +Pod readiness gates rely on [»admission webhooks«](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), where the Kubernetes API server makes calls to the AWS Gateway API controller as part of pod creation. This call is made using TLS, so the controller must present a TLS certificate. This certificate is stored as a standard Kubernetes secret. If you are using Helm, the certificate will automatically be configured as part of the Helm install. + +If you are manually deploying the controller, for example using the ```deploy.yaml``` file, you will need to create the tls secret for the webhook in the controller namespace. The ```deploy.yaml``` file includes a placeholder secret, but it must be updated if you wish to use the webhook. The placeholder secret _will not_ pass API server validations, but will ensure the controller container is able to start. + +### Webhook secret requirements +The webhook requires a specific kubernetes secret to exist in the same namespace as the webhook itself: +* secret name: ```webhook-cert``` +* default controller namespace: ```aws-application-networking-system``` +```console +# example create-secret command, assumes tls.crt and tls.key exist in current directory +# if the placeholder secret exists, you will need to delete it before setting the new value +kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.crt --key=tls.key +``` + +### Webhook secret configuration example +The below example creates an unsigned certificate, adds it as the webhook secret, then patches the webhook configuration so the API server trusts the certificate. + +If your cluster uses its own PKI and includes appropriate trust configuration for the API server, the certificate issued would be signed by your internal certificate authority and therefore not require the ```kubectl patch``` command below. +```console +# Example commands to configure the webhook to use an unsigned certificate +CERT_FILE=tls.crt +KEY_FILE=tls.key + +WEBHOOK_SVC_NAME=webhook-service +WEBHOOK_NAME=aws-appnet-gwc-mutating-webhook +WEBHOOK_NAMESPACE=aws-application-networking-system +WEBHOOK_SECRET_NAME=webhook-cert + +# Step 1: generate a certificate if needed, can also be provisioned through orgnanizational PKI, etc +# This cert includes a 100 year expiry +HOST=${WEBHOOK_SVC_NAME}.${WEBHOOK_NAMESPACE}.svc +openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" \ + -addext "subjectAltName = DNS:${HOST}, DNS:${HOST}.cluster.local" + +# Step 2: replace the placeholder secret from deploy.yaml +kubectl delete secret $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE +kubectl create secret tls $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE --cert=${CERT_FILE} --key=${KEY_FILE} + +# Step 3: Patch the webhook CA bundle to exactly the cert being used. +# This will ensure Kubernetes API server is able to trust the certificate presented by the webhook. +# This step would not be required if you are using a signed certificate that is already trusted by the API server +CERT_B64=$(cat tls.crt | base64) +kubectl patch mutatingwebhookconfigurations.admissionregistration.k8s.io $WEBHOOK_NAME \ + --namespace $WEBHOOK_NAMESPACE --type='json' \ + -p="[{'op': 'replace', 'path': '/webhooks/0/clientConfig/caBundle', 'value': '${CERT_B64}'}]" +``` + +## Configuration +Pod readiness gate support is enabled by default on the AWS Gateway API controller. To enable the feature, you must apply a label to each of the namespaces you would like to use this feature. You can create and label a namespace as follows - + +``` +$ kubectl create namespace example-ns +namespace/example-ns created + +$ kubectl label namespace example-ns application-networking.k8s.aws/pod-readiness-gate-inject=enabled +namespace/example-ns labeled + +$ kubectl describe namespace example-ns +Name: example-ns +Labels: application-networking.k8s.aws/pod-readiness-gate-inject=enabled + kubernetes.io/metadata.name=example-ns +Annotations: +Status: Active +``` + +Once labelled, the controller will add the pod readiness gates to all subsequently created pods. + +The readiness gates have the condition type ```application-networking.k8s.aws/pod-readiness-gate``` and the controller injects the config to the pod spec only during pod creation. + +## Object Selector +The default webhook configuration matches all pods in the namespaces containing the label `application-networking.k8s.aws/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select ONLY pods with `application-networking.k8s.aws/pod-readiness-gate-inject: enabled` label instead of all pods in the labeled namespace, you can add the following `objectSelector` to the webhook: +``` + objectSelector: + matchLabels: + application-networking.k8s.aws/pod-readiness-gate-inject: enabled +``` +To edit, +``` +$ kubectl edit mutatingwebhookconfigurations aws-appnet-gwc-mutating-webhook + ... + name: mpod.gwc.k8s.aws + namespaceSelector: + matchExpressions: + - key: application-networking.k8s.aws/pod-readiness-gate-inject + operator: In + values: + - enabled + objectSelector: + matchLabels: + application-networking.k8s.aws/pod-readiness-gate-inject: enabled + ... +``` +When you specify multiple selectors, pods matching all the conditions will get mutated. + +## Checking the pod condition status + +The status of the readiness gates can be verified with `kubectl get pod -o wide`: +``` +NAME READY STATUS RESTARTS AGE IP NODE READINESS GATES +nginx-test-5744b9ff84-7ftl9 1/1 Running 0 81s 10.1.2.3 ip-10-1-2-3.ec2.internal 0/1 +``` + +When the target is registered and healthy in the VPC Lattice target group, the output will look like: +``` +NAME READY STATUS RESTARTS AGE IP NODE READINESS GATES +nginx-test-5744b9ff84-7ftl9 1/1 Running 0 81s 10.1.2.3 ip-10-1-2-3.ec2.internal 1/1 +``` + +If a readiness gate doesn't get ready, you can check the reason via: + +```console +$ kubectl get pod nginx-test-545d8f4d89-l7rcl -o yaml | grep -B7 'type: application-networking.k8s.aws/pod-readiness-gate' +status: + conditions: + - lastProbeTime: null + lastTransitionTime: null + reason: HEALTHY + status: "True" + type: application-networking.k8s.aws/pod-readiness-gate +``` diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 07a53b28..133bd3a6 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -25,6 +25,7 @@ const ( DEFAULT_SERVICE_NETWORK = "DEFAULT_SERVICE_NETWORK" ENABLE_SERVICE_NETWORK_OVERRIDE = "ENABLE_SERVICE_NETWORK_OVERRIDE" AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" + DEV_MODE = "DEV_MODE" ) var VpcID = "" @@ -32,6 +33,7 @@ var AccountID = "" var Region = "" var DefaultServiceNetwork = "" var ClusterName = "" +var DevMode = "" var ServiceNetworkOverrideMode = false @@ -44,7 +46,8 @@ func ConfigInit() error { func configInit(sess *session.Session, metadata EC2Metadata) error { var err error - // CLUSTER_VPC_ID + DevMode = os.Getenv(DEV_MODE) + VpcID = os.Getenv(CLUSTER_VPC_ID) if VpcID == "" { VpcID, err = metadata.VpcID() @@ -53,7 +56,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // REGION Region = os.Getenv(REGION) if Region == "" { Region, err = metadata.Region() @@ -62,7 +64,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // AWS_ACCOUNT_ID AccountID = os.Getenv(AWS_ACCOUNT_ID) if AccountID == "" { AccountID, err = metadata.AccountId() @@ -71,7 +72,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { } } - // DEFAULT_SERVICE_NETWORK DefaultServiceNetwork = os.Getenv(DEFAULT_SERVICE_NETWORK) overrideFlag := os.Getenv(ENABLE_SERVICE_NETWORK_OVERRIDE) @@ -79,7 +79,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { ServiceNetworkOverrideMode = true } - // CLUSTER_NAME ClusterName, err = getClusterName(sess) if err != nil { return fmt.Errorf("cannot get cluster name: %s", err) diff --git a/pkg/webhook/pod_mutator.go b/pkg/webhook/pod_mutator.go new file mode 100644 index 00000000..e3f13346 --- /dev/null +++ b/pkg/webhook/pod_mutator.go @@ -0,0 +1,48 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/webhook/core" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + apiPathMutatePod = "/mutate-pod" +) + +func NewPodMutator(scheme *runtime.Scheme, podReadinessGateInjector *PodReadinessGateInjector) *podMutator { + return &podMutator{ + podReadinessGateInjector: podReadinessGateInjector, + scheme: scheme, + } +} + +var _ core.Mutator = &podMutator{} + +type podMutator struct { + podReadinessGateInjector *PodReadinessGateInjector + scheme *runtime.Scheme +} + +func (m *podMutator) Prototype(_ admission.Request) (runtime.Object, error) { + return &corev1.Pod{}, nil +} + +func (m *podMutator) MutateCreate(ctx context.Context, obj runtime.Object) (runtime.Object, error) { + pod := obj.(*corev1.Pod) + if err := m.podReadinessGateInjector.Mutate(ctx, pod); err != nil { + return pod, err + } + return pod, nil +} + +func (m *podMutator) MutateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { + return obj, nil +} + +func (m *podMutator) SetupWithManager(mgr ctrl.Manager) { + mgr.GetWebhookServer().Register(apiPathMutatePod, core.MutatingWebhookForMutator(m.scheme, m)) +} diff --git a/pkg/webhook/pod_mutator_test.go b/pkg/webhook/pod_mutator_test.go new file mode 100644 index 00000000..1c899f98 --- /dev/null +++ b/pkg/webhook/pod_mutator_test.go @@ -0,0 +1,81 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestReadinessGateInjectionNew(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + pod := &corev1.Pod{} + + ret, err := m.MutateCreate(context.TODO(), pod) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 1, len(newPod.Spec.ReadinessGates)) + ct := newPod.Spec.ReadinessGates[0].ConditionType + assert.Equal(t, PodReadinessGateConditionType, string(ct)) +} + +func TestReadinessGateAlreadyExists(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + pod := &corev1.Pod{} + prg := corev1.PodReadinessGate{ConditionType: PodReadinessGateConditionType} + pod.Spec.ReadinessGates = append(pod.Spec.ReadinessGates, prg) + + ret, err := m.MutateCreate(context.TODO(), pod) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 1, len(newPod.Spec.ReadinessGates)) + ct := newPod.Spec.ReadinessGates[0].ConditionType + assert.Equal(t, PodReadinessGateConditionType, string(ct)) +} + +func TestUpdateDoesNothing(t *testing.T) { + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + + k8sClient := testclient. + NewClientBuilder(). + WithScheme(k8sScheme). + Build() + + injector := NewPodReadinessGateInjector(k8sClient, gwlog.FallbackLogger) + m := NewPodMutator(k8sScheme, injector) + + p1 := &corev1.Pod{} + p1.Spec.Hostname = "foo" + p2 := &corev1.Pod{} + p2.Spec.Hostname = "bar" + + ret, err := m.MutateUpdate(context.TODO(), p1, p2) + newPod := ret.(*corev1.Pod) + assert.Nil(t, err) + assert.Equal(t, 0, len(newPod.Spec.ReadinessGates)) +} diff --git a/pkg/webhook/pod_readiness_gate_injector.go b/pkg/webhook/pod_readiness_gate_injector.go new file mode 100644 index 00000000..3615a121 --- /dev/null +++ b/pkg/webhook/pod_readiness_gate_injector.go @@ -0,0 +1,43 @@ +package webhook + +import ( + "context" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + PodReadinessGateConditionType = "application-networking.k8s.aws/pod-readiness-gate" +) + +func NewPodReadinessGateInjector(k8sClient client.Client, log gwlog.Logger) *PodReadinessGateInjector { + return &PodReadinessGateInjector{ + k8sClient: k8sClient, + log: log, + } +} + +type PodReadinessGateInjector struct { + k8sClient client.Client + log gwlog.Logger +} + +func (m *PodReadinessGateInjector) Mutate(ctx context.Context, pod *corev1.Pod) error { + pct := corev1.PodConditionType(PodReadinessGateConditionType) + m.log.Debugf("Webhook invoked for pod %s/%s", pod.Name, pod.Namespace) + + found := false + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + found = true + break + } + } + if !found { + pod.Spec.ReadinessGates = append(pod.Spec.ReadinessGates, corev1.PodReadinessGate{ + ConditionType: pct, + }) + } + return nil +} diff --git a/test/suites/webhook/readiness_gate_inject_test.go b/test/suites/webhook/readiness_gate_inject_test.go new file mode 100644 index 00000000..861cefab --- /dev/null +++ b/test/suites/webhook/readiness_gate_inject_test.go @@ -0,0 +1,89 @@ +package webhook + +import ( + "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/webhook" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "time" +) + +var _ = Describe("Readiness Gate Inject", Ordered, func() { + untaggedNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-e2e-test-no-tag", + }, + } + taggedNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-e2e-test-tagged", + Labels: map[string]string{ + "application-networking.k8s.aws/pod-readiness-gate-inject": "enabled", + }, + }, + } + + BeforeAll(func() { + Eventually(func(g Gomega) { + _ = testFramework.Delete(ctx, untaggedNS) + _ = testFramework.Delete(ctx, taggedNS) + testFramework.EventuallyExpectNotFound(ctx, untaggedNS, taggedNS) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + + Eventually(func(g Gomega) { + testFramework.ExpectCreated(ctx, untaggedNS, taggedNS) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + }) + + It("create deployment in untagged namespace, no readiness gate", func() { + deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "untagged-test-pod", Namespace: untaggedNS.Name}) + Eventually(func(g Gomega) { + testFramework.Create(ctx, deployment) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + g.Expect(len(pods)).To(BeEquivalentTo(1)) + + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + g.Expect(true).To(BeFalse(), "Pod readiness gate was injected to unlabeled namespace") + } + } + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + }) + + It("create deployment in tagged namespace, has readiness gate", func() { + deployment, _ := testFramework.NewHttpApp(test.HTTPAppOptions{Name: "tagged-test-pod", Namespace: taggedNS.Name}) + Eventually(func(g Gomega) { + testFramework.Create(ctx, deployment) + testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) + + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + g.Expect(len(pods)).To(BeEquivalentTo(1)) + + pod := pods[0] + pct := corev1.PodConditionType(webhook.PodReadinessGateConditionType) + + foundCount := 0 + for _, rg := range pod.Spec.ReadinessGates { + if rg.ConditionType == pct { + foundCount++ + } + } + + g.Expect(foundCount).To(Equal(1), + fmt.Sprintf("Pod readiness gate was expected on labeled namespace. Found %d times", foundCount)) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + }) + + AfterAll(func() { + testFramework.ExpectDeletedThenNotFound(ctx, untaggedNS, taggedNS) + }) +}) diff --git a/test/suites/webhook/suite_test.go b/test/suites/webhook/suite_test.go new file mode 100644 index 00000000..1e9cdadd --- /dev/null +++ b/test/suites/webhook/suite_test.go @@ -0,0 +1,40 @@ +package webhook + +import ( + "context" + "os" + + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/zap" + + "testing" +) + +const ( + k8snamespace = "webhook-" + test.K8sNamespace +) + +var testFramework *test.Framework +var ctx context.Context + +var _ = SynchronizedBeforeSuite(func() { + vpcId := os.Getenv("CLUSTER_VPC_ID") + if vpcId == "" { + Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") + } +}, func() { +}) + +func TestIntegration(t *testing.T) { + ctx = test.NewContext(t) + logger := gwlog.NewLogger(zap.DebugLevel) + testFramework = test.NewFramework(ctx, logger, k8snamespace) + RegisterFailHandler(Fail) + RunSpecs(t, "WebhookIntegration") +} + +var _ = SynchronizedAfterSuite(func() {}, func() { +})