diff --git a/README.md b/README.md index 09fc2b6..afbc873 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,77 @@ spec: The `ozctl` tool provides end-users with a quick and easy way to request access against pre-defined access templates. T + +## Architecture + +The **Oz** controller operates using the standard +[controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) +framework. To help better explain the flow that users and operators of this +tool can expect, we've got some architecture diagrams below. For more detailed +diagrams of the internal workings, see the +[`controllers/README.md`](controllers/README.md) document. + + +### How `ozctl` and **Oz** work together for a `PodAccessRequest` + +In the simple scenario where an engineer _Alice_ needs a temporary Pod created +to perform some work, here's the basic flow: + +```mermaid +sequenceDiagram + participant Alice + participant Ozctl + participant Kubernetes + participant Oz + participant PodAccessRequest + + link PodAccessRequest: API @ [pod_access_request] + + Note over Alice,Ozctl: Alice requests access to a development Pod + Alice->>Ozctl: ozctl create podaccessrequest + + Note over Ozctl,Kubernetes: CLI prepares a PodAccessRequest{} resource + Ozctl->>Kubernetes: Create PodAccessRequest{}... + + Note over Kubernetes,Oz: Mutating Webhook called... + Kubernetes->>Oz: /mutate-v1-pod... + Oz-->Oz: Call Default(admission.Request) + + Note over Kubernetes,Oz: Mutated PodAccessRequest is returned + Oz->>Kubernetes: User Info Context applied + + Note over Kubernetes,Oz: Validating Webhook called to record Alice's action + Kubernetes->>Oz: /validate-v1-pod... + + Note over Kubernetes,Oz: Emit Log Event + Oz-->Oz: Call ValidateCreate(...) + Oz-->Oz: Call Log.Info("Alice ...") + Oz->>Kubernetes: `Allowed=True` + + Note over Kubernetes,Ozctl: Cluster responds that the resource has been created + Kubernetes->>Ozctl: PodAccessRequest{} created + + par + loop Reconcile Loop... + Note over Kubernetes,Oz: Initial trigger event from Kubernetes + Kubernetes->>Oz: Reconcile(PodAccessRequest) + + Oz-->Oz: Verify Request Durations + Oz-->Oz: Verify Access Still Valid + Oz->>Kubernetes: Create Role, RoleBinding, Pod + Kubernetes ->> Oz: Resources Created + Oz-->Oz: Verify Pod is "Ready" + Oz->>Kubernetes: Set Status.IsReady=True + end + and + loop CLI Loop + Ozctl->>Kubernetes: Is Status.IsReady? + Kubernetes->>Ozctl: True + Ozctl->>Alice: "You're ready... kubectl exec ..." + end + end +``` + ## License Copyright 2022 Matt Wise. diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index a6fe5da..dabf32f 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -94,3 +94,25 @@ webhooks: resources: - podaccessrequests sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /watch-v1-pod + failurePolicy: Fail + name: vpod.kb.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - CONNECT + resources: + - pods/exec + - pods/attach + sideEffects: None diff --git a/controllers/README.md b/controllers/README.md new file mode 100644 index 0000000..828e8ac --- /dev/null +++ b/controllers/README.md @@ -0,0 +1,202 @@ +[exec_access_request]: /API.md#execaccessrequest +[exec_access_template]: /API.md#execaccesstemplate +[pod_access_request]: /API.md#podaccessrequest +[pod_access_template]: /API.md#podaccesstemplate +[access_config]: /API.md#accessconfig +[target_ref]: /API.md#crossversionobjectreference +[builders]: ./builders/README.md +[runtime]: https://github.com/kubernetes-sigs/controller-runtime + +# Controllers + +The Controllers in this package leverage the [controller-runtime][runtime] +package to define controllers that handle our custom resources +([PodAccessRequest][pod_access_request], +[PodAccessTemplate][pod_access_template], +[ExecAccessRequest][exec_access_request], +[ExecAccessTemplate][exec_access_request]). There are also controllers in this +package that handle inbound webhooks via the [Admission +Controllers](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) +system. + +## Reconcilers + +Our `Reconciler` controllers handle operating in a loop to ensure that our +Custom Resources are consistently in the desired state. These controllers all +implement a `reconcile()` function that is triggered by `Watch...` requests +against the Kubernetes API. + +Generally speaking, we try to keep the `reconcile()` functions short and easy +to read/understand. The heavy lifting is actually done by our +[`Builder`][builders] structs. + +## [`ExecAccessTemplateReconciler`](exec_access_template_controller.go) + +The [`ExecAccessTemplateReconciler`](exec_access_template_controller.go) is a +very simple controller whose job is to make sure that the `ExecAccessTemplate` +is valid and available for use. It primarily validates that the template has +valid [`AccessConfig`][access_config] settings, and a valid +[`TargetRef`][target_ref] pointing to a real Pod controller (Deployment, etc). + +```mermaid +sequenceDiagram + participant Kubernetes + participant Oz + participant ExecAccessTemplateReconciler + + Note over Oz,Kubernetes: The Oz Controller begins to watch for resources + Oz->>Kubernetes: Watch ExecAccessTemplate{} Resources... + Kubernetes->>Oz: New ExecAccessTemplate{} Created + + loop Reconcile Loop... + Note over Oz,ExecAccessTemplateReconciler: Runtime calls Reconciler function + Oz->>ExecAccessTemplateReconciler: reconcile(...) + + Note over ExecAccessTemplateReconciler: Verify Target Reference Exists + ExecAccessTemplateReconciler->>Kubernetes: Get Deployment{Name: foo} + Kubernetes->>ExecAccessTemplateReconciler: + + Note over ExecAccessTemplateReconciler: Verify Access Configurations Settings are Valid + ExecAccessTemplateReconciler-->ExecAccessTemplateReconciler: api.VerifyMiscSettings() + + Note over ExecAccessTemplateReconciler: Write ready state back into resource + ExecAccessTemplateReconciler->>Kubernetes: Update .Status.IsReady=True + end +``` + +## [`ExecAccessRequestReconciler`](exec_access_request_controller.go) + +The [`ExecAccessRequestReconciler`](exec_access_request_controller.go) handles +creating a `Role` and `RoleBinding` that grant an engineer `kubectl exec ...` +access into an already existing Pod for a particular target deploymnt. + +The reconciler logic itself is fairly simple, and most of the heavy lifting is +actually handled by a [`ExecAccessBuilder`](builders/exec_access_builder.go). + +```mermaid +sequenceDiagram + participant Kubernetes + participant Oz + participant ExecAccessRequestReconciler + participant ExecAccessBuilder + participant ExecAccessTemplate + + Oz->>Kubernetes: Watch ExecAccessRequest{} Resources... + Kubernetes->>Oz: New ExecAccessRequest{} Created + + loop Reconcile Loop... + Note over Oz,ExecAccessRequestReconciler: Runtime calls Reconciler function + Oz-->>ExecAccessRequestReconciler: reconcile(...) + + Note over ExecAccessRequestReconciler: Verify `ExecAccessTemplate` Exists + ExecAccessRequestReconciler->>Kubernetes: Get ExecAccessTemplate{Name: foo} + Kubernetes->>ExecAccessRequestReconciler: + + Note over ExecAccessRequestReconciler: Verify AccessConfiguration Settings are Valid + ExecAccessRequestReconciler-->>ExecAccessRequestReconciler: verifyDuration() + ExecAccessRequestReconciler-->>ExecAccessRequestReconciler: isAccessExpired() + + Note over ExecAccessRequestReconciler,ExecAccessBuilder: Begin Building Access Resources + ExecAccessRequestReconciler-->>ExecAccessBuilder: verifyAccessResourcesBuilt() + + ExecAccessBuilder->>Kubernetes: Get Deployment{Name: foo..} + Kubernetes->>ExecAccessBuilder: + + Note over ExecAccessBuilder: Create the Resources + ExecAccessBuilder->>Kubernetes: Create Role{Name: foo...} + ExecAccessBuilder->>Kubernetes: Create RoleBinding{Name: foo...} + + Note over ExecAccessRequestReconciler: Write ready state back into resource + ExecAccessRequestReconciler->>Kubernetes: Update .Status.IsReady=True + end +``` + +## [`PodAccessTemplateReconciler`](pod_access_template_controller.go) + +The [`PodAccessTemplateReconciler`](pod_access_template_controller.go) is a +very simple controller whose job is to make sure that the `PodAccessTemplate` +is valid and available for use. It primarily validates that the template has +valid [`AccessConfig`][access_config] settings, and a valid +[`TargetRef`][target_ref] pointing to a real Pod controller (Deployment, etc). + +```mermaid +sequenceDiagram + participant Kubernetes + participant Oz + participant PodAccessTemplateReconciler + + Note over Oz,Kubernetes: The Oz Controller begins to watch for resources + Oz->>Kubernetes: Watch PodAccessTemplate{} Resources... + Kubernetes->>Oz: New PodAccessTemplate{} Created + + loop Reconcile Loop... + Note over Oz,PodAccessTemplateReconciler: Runtime calls Reconciler function + Oz->>PodAccessTemplateReconciler: reconcile(...) + + Note over PodAccessTemplateReconciler: Verify Target Reference Exists + PodAccessTemplateReconciler->>Kubernetes: Get Deployment{Name: foo} + Kubernetes->>PodAccessTemplateReconciler: + + Note over PodAccessTemplateReconciler: Verify Access Configurations Settings are Valid + PodAccessTemplateReconciler-->PodAccessTemplateReconciler: api.VerifyMiscSettings() + + Note over PodAccessTemplateReconciler: Write ready state back into resource + PodAccessTemplateReconciler->>Kubernetes: Update .Status.IsReady=True + end +``` + +## [`PodAccessRequestReconciler`](pod_access_request_controller.go) + +The [`PodAccessRequestReconciler`](pod_access_request_controller.go) handles +the creation of a dedicated workload `Pod` for an engineer on-demand based on +the configuration of a [`PodAccessTemplate`](#podaccesstemplatereconciler). The +reconciler logic itself is fairly simple, and most of the heavy lifting is +actually handled by a [`PodAccessBuilder`](builders/pod_access_builder.go). + +```mermaid +sequenceDiagram + participant Kubernetes + participant Oz + participant PodAccessRequestReconciler + participant PodAccessBuilder + participant PodAccessTemplate + + Oz->>Kubernetes: Watch PodAccessRequest{} Resources... + Kubernetes->>Oz: New PodAccessRequest{} Created + + loop Reconcile Loop... + Note over Oz,PodAccessRequestReconciler: Runtime calls Reconciler function + Oz-->>PodAccessRequestReconciler: reconcile(...) + + Note over PodAccessRequestReconciler: Verify `PodAccessTemplate` Exists + PodAccessRequestReconciler->>Kubernetes: Get PodAccessTemplate{Name: foo} + Kubernetes->>PodAccessRequestReconciler: + + Note over PodAccessRequestReconciler: Verify AccessConfiguration Settings are Valid + PodAccessRequestReconciler-->>PodAccessRequestReconciler: verifyDuration() + PodAccessRequestReconciler-->>PodAccessRequestReconciler: isAccessExpired() + + Note over PodAccessRequestReconciler,PodAccessBuilder: Begin Building Access Resources + PodAccessRequestReconciler-->>PodAccessBuilder: verifyAccessResourcesBuilt() + + PodAccessBuilder->>Kubernetes: Get Deployment{Name: foo..} + Kubernetes->>PodAccessBuilder: + PodAccessBuilder-->>PodAccessTemplate: GenerateMutatedPodSpec(Deployment{}...) + + Note over PodAccessBuilder: Create the Resources + PodAccessBuilder->>Kubernetes: Create Pod{Name: foo...} + PodAccessBuilder->>Kubernetes: Create Role{Name: foo...} + PodAccessBuilder->>Kubernetes: Create RoleBinding{Name: foo...} + + + Note over PodAccessBuilder: Verify Resources Ready + PodAccessRequestReconciler-->>PodAccessBuilder: verifyAccessResourcesReady() + + PodAccessBuilder->>Kubernetes: Get Pod{}.Status.Ready + Kubernetes->>PodAccessBuilder: Pod{}.Status.Ready=True + PodAccessBuilder-->>PodAccessRequestReconciler: Pod Is Ready + + Note over PodAccessRequestReconciler: Write ready state back into resource + PodAccessRequestReconciler->>Kubernetes: Update .Status.IsReady=True + end +``` \ No newline at end of file diff --git a/controllers/builders/README.md b/controllers/builders/README.md new file mode 100644 index 0000000..9468111 --- /dev/null +++ b/controllers/builders/README.md @@ -0,0 +1 @@ +.... \ No newline at end of file diff --git a/controllers/pod_access_request_controller.go b/controllers/pod_access_request_controller.go index 28b22f8..635d0b1 100644 --- a/controllers/pod_access_request_controller.go +++ b/controllers/pod_access_request_controller.go @@ -72,6 +72,8 @@ func (r *PodAccessRequestReconciler) Reconcile( // First make sure we use the ApiReader (non-cached) client to go and figure out if the resource exists or not. If // it doesn't come back, we exit out beacuse it is likely the object has been deleted and we no longer need to // worry about it. + // + // TODO: Validate IsReady(). logger.Info("Verifying PodAccessRequest exists") resource, err := api.GetPodAccessRequest(ctx, r.APIReader, req.Name, req.Namespace) if err != nil { diff --git a/controllers/pod_watcher.go b/controllers/pod_watcher.go new file mode 100644 index 0000000..f7f80c6 --- /dev/null +++ b/controllers/pod_watcher.go @@ -0,0 +1,78 @@ +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// example code: https://github.com/kubernetes-sigs/controller-runtime/blob/master/examples/builtins/validatingwebhook.go + +// PodExecWatcher is a ValidatingWebhookEndpoint that receives calls from the +// Kubernetes API just before Pod's "exec" subresource is written into the +// cluster. The intention for this resource is to perform audit-logging type +// actions in the short term, and in the long term provide a more granular +// layer of security for Pod Exec access. +type PodExecWatcher struct { + Client client.Client + decoder *admission.Decoder +} + +// +kubebuilder:webhook:path=/watch-v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,groups="",resources=pods/exec;pods/attach,verbs=create;update;connect,versions=v1,name=vpod.kb.io,admissionReviewVersions=v1 + +// Handle logs out each time an Exec/Attach call is made on a pod. +// +// Right now this is purely an informative log event. When we take care of +// https://github.com/diranged/oz/issues/24, we can use this handler to push +// events onto the Pods (and Access Requests) for audit purposes. +// +// Additionally, https://github.com/diranged/oz/issues/50 and +// https://github.com/diranged/oz/issues/51 will be handled through this +// endpoint in the future. +func (w *PodExecWatcher) Handle(ctx context.Context, req admission.Request) admission.Response { + logger := log.FromContext(ctx) + logger.Info( + fmt.Sprintf( + "Handling %s Operation on %s/%s by %s", + req.Operation, + req.Resource.Resource, + req.Name, + req.UserInfo.Username, + ), "request", ObjectToJSON(req), + ) + + exec := &corev1.PodExecOptions{} + err := w.decoder.Decode(req, exec) + if err != nil { + logger.Error(err, "Couldnt decode") + return admission.Errored(http.StatusBadRequest, err) + } + + return admission.Allowed("") +} + +// PodWatcher implements admission.DecoderInjector. +// A decoder will be automatically injected. + +// InjectDecoder injects the decoder. +func (w *PodExecWatcher) InjectDecoder(d *admission.Decoder) error { + w.decoder = d + return nil +} + +// ObjectToJSON is a quick helper function for pretty-printing an entire K8S object in JSON form. +// Used in certain debug log statements primarily. +func ObjectToJSON(obj any) string { + jsonData, err := json.Marshal(obj) + if err != nil { + fmt.Printf("could not marshal json: %s\n", err) + return "" + } + return string(jsonData) +} diff --git a/controllers/pod_watcher_test.go b/controllers/pod_watcher_test.go new file mode 100644 index 0000000..b758e39 --- /dev/null +++ b/controllers/pod_watcher_test.go @@ -0,0 +1,110 @@ +package controllers + +import ( + "context" + "encoding/json" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var _ = Describe("PodWatcher", Ordered, func() { + decoder, _ := admission.NewDecoder(scheme.Scheme) + + Context("Functional Unit Tests", func() { + var ( + admissionRequest *admission.Request + ctx = context.Background() + requestName = "test" + watcher = &PodExecWatcher{Client: k8sClient, decoder: decoder} + request = &corev1.PodExecOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + Stdin: false, + Stdout: false, + Stderr: false, + TTY: false, + Container: "", + Command: []string{}, + } + resource = metav1.GroupVersionResource{ + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Resource: corev1.Pod{}.Kind, + } + kind = metav1.GroupVersionKind{ + Group: api.SchemeGroupVersion.Group, + Version: api.SchemeGroupVersion.Version, + Kind: corev1.PodExecOptions{}.Kind, + } + ) + + It("Handle() calls with invalid data should return an error", func() { + admissionRequest = &admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Resource: resource, + RequestKind: &kind, + RequestResource: &resource, + Name: requestName, + Namespace: requestName, + Operation: "CONNET", + UserInfo: authenticationv1.UserInfo{ + Username: "admin", + UID: "", + Groups: []string{}, + Extra: map[string]authenticationv1.ExtraValue{ + "": {}, + }, + }, + Object: runtime.RawExtension{ + Raw: []byte("asdf"), + }, + }, + } + resp := watcher.Handle(ctx, *admissionRequest) + Expect(resp.Result.Code).To(Equal(int32(http.StatusBadRequest))) + Expect(resp.Allowed).To(BeFalse()) + }) + + // TODO: The "Userinfo" checks should move into an authentication + // package so that we can write one set of tests for all of the + // Validate* functions. + It("Create calls with req.Userinfo{} should suceed", func() { + requestBytes, _ := json.Marshal(request) + admissionRequest = &admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Resource: resource, + RequestKind: &kind, + RequestResource: &resource, + Name: requestName, + Namespace: requestName, + Operation: "CONNET", + UserInfo: authenticationv1.UserInfo{ + Username: "admin", + UID: "", + Groups: []string{}, + Extra: map[string]authenticationv1.ExtraValue{ + "": {}, + }, + }, + Object: runtime.RawExtension{ + Raw: requestBytes, + }, + }, + } + resp := watcher.Handle(ctx, *admissionRequest) + Expect(resp.Allowed).To(BeTrue()) + }) + }) +}) diff --git a/main.go b/main.go index 9cf0308..59edc3f 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook" zaplogfmt "github.com/jsternberg/zap-logfmt" @@ -127,6 +128,34 @@ func main() { os.Exit(1) } + // Webhooks for our core CRDs are registered through the api/v1alpha1 + // package. These webhooks are registered so that we can pre-populate (or + // validate) our custom resources before they ever get to the Reconcile() + // functions. + if err = (&crdsv1alpha1.PodAccessRequest{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "PodAccessRequest") + os.Exit(1) + } + if err = (&crdsv1alpha1.ExecAccessRequest{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "ExecAccessRequest") + os.Exit(1) + } + + // These special Webhooks are registered for the purpose of event-logging + // user-actions. + hookServer := mgr.GetWebhookServer() + hookServer.Register( + "/watch-v1-pod", + &webhook.Admission{Handler: &controllers.PodExecWatcher{Client: mgr.GetClient()}}, + ) + + // Set Up the Reconcilers + // + // These are the core components that are "watching" the custom resource + // (PodAccessRequests, PodAccessTemplates, etc). These reconcilers may + // depend on some information having been injected by the Webhooks we + // registered above. + // if err = (&controllers.ExecAccessTemplateReconciler{ BaseTemplateReconciler: controllers.BaseTemplateReconciler{ BaseReconciler: controllers.BaseReconciler{ @@ -183,16 +212,6 @@ func main() { os.Exit(1) } - // Webhooks - if err = (&crdsv1alpha1.PodAccessRequest{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "PodAccessRequest") - os.Exit(1) - } - if err = (&crdsv1alpha1.ExecAccessRequest{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "ExecAccessRequest") - os.Exit(1) - } - //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {