oci-hook: allow users to set namespace exceptions and define default

Prior to this commit `kube-system` was always added as an exception in order for Pods in that namespace to be created in the event that the OCI hook was not able to reach the Tetragon agent.
This leads to a deadlock scenario when Tetragon itself is not installed in `kube-system` as the agent won't start and leave the cluster in an unhealthy state.

Instead of always adding `kube-syste` this change will always add the namespace Tetragon is deployed in, to guarantee that the agent can always start.
In addition to that a user can now define additional namespaces as further exceptions.
For example to ensure Pods in business-critical namespaces can still be created even if the OCI hook fails to reach the Tetragon agent.

Signed-off-by: Filip Nikolic <oss.filipn@gmail.com>

contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go

@@ -28,7 +28,8 @@ type Install struct {
 	HostInstallDir  string `required help:"Installation dir (in the host). Used for the binary and the hook logfile."`
 
 	OciHooks struct {
-		LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"`
+		LocalDir            string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"`
+		FailAllowNamespaces string `help:"Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent."`
 	} `embed:"" prefix:"oci-hooks."`
 }
 
@@ -60,7 +61,7 @@ func (i *Install) ociHooksInstall(log *logrus.Logger) {
 	binFname := filepath.Join(i.HostInstallDir, binBaseName)
 
 	logFname := filepath.Join(i.HostInstallDir, logBaseName)
-	hook := ociHooksConfig(binFname, "--log-fname", logFname)
+	hook := ociHooksConfig(binFname, "--log-fname", logFname, "--fail-allow-namespaces", i.OciHooks.FailAllowNamespaces)
 	data, err := json.MarshalIndent(hook, "", "   ")
 	if err != nil {
 		log.WithError(err).Fatal("failed to unmarshall hook info")

contrib/rthooks/tetragon-oci-hook/docs/demo.md

@@ -29,7 +29,7 @@ helm install --namespace kube-system \
         --set tetragon.image.override=localhost/cilium/tetragon:latest  \
         --set tetragon.grpc.address="unix:///var/run/cilium/tetragon/tetragon.sock" \
         --set tetragon.ociHookSetup.enabled=true \
-        tetragon ./install/kubernetes
+        tetragon ./install/kubernetes/tetragon
 ...
 kubectl logs -n kube-system tetragon-289tf  -c oci-hook-setup
 time="2023-12-05T09:28:50Z" level=info msg="written binary" hook-dst-path=/hostInstall/tetragon-oci-hook

contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: tetragon-oci-hook-uninstall
-  namespace: kube-system
   labels:
     k8s-app: tetragon-oci-hook-setup-test
 spec:

install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl

@@ -11,6 +11,7 @@
     - --local-install-dir={{  include "container.tetragonOCIHookSetup.installPath" . }}
     - --host-install-dir={{ .Values.tetragon.ociHookSetup.installDir }}
     - --oci-hooks.local-dir={{ include "container.tetragonOCIHookSetup.hooksPath" . }}
+    - --oci-hooks.fail-allow-namespaces={{ if .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ printf "%s,%s" .Release.Namespace .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ else }}{{ .Release.Namespace }}{{ end }}
   volumeMounts:
     {{- with .Values.tetragon.ociHookSetup.extraVolumeMounts }}
       {{- toYaml . | nindent 4 }}

install/kubernetes/tetragon/values.yaml

@@ -200,6 +200,9 @@ tetragon:
     # "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md).
     interface: "oci-hooks"
     installDir: "/opt/tetragon"
+    # -- Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent.
+    # The namespace Tetragon is deployed in is always added as an exception and must not be added again.
+    failAllowNamespaces: ""
     # -- Security context for oci-hook-setup init container
     securityContext:
       privileged: true