diff --git a/operator/kuttl-test.yaml b/operator/kuttl-test.yaml index 60e2fee679742..f95e2b4632494 100644 --- a/operator/kuttl-test.yaml +++ b/operator/kuttl-test.yaml @@ -2,6 +2,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestSuite testDirs: - ./tests/central +- ./tests/controller - ./tests/securedcluster # central pod takes a while do become healthy, typically ~25s, but sometimes over a minute since container start. # We need to add CR reconciliation, pod creation and image fetch to that. diff --git a/operator/tests/TROUBLESHOOTING_E2E_TESTS.md b/operator/tests/TROUBLESHOOTING_E2E_TESTS.md index 0510e2fd8cd63..26fae63978471 100644 --- a/operator/tests/TROUBLESHOOTING_E2E_TESTS.md +++ b/operator/tests/TROUBLESHOOTING_E2E_TESTS.md @@ -2,7 +2,8 @@ ## Structure -All kuttl tests are in the [tests](.) directory. It contains four tests, contained in three suites: +All kuttl tests are in the [tests](.) directory. It contains the following tests, grouped in four suites: +- [controller/metrics](controller/metrics) - checks authenticated access to stackrox operator controller manager metrics - [central/central-basic](central/central-basic) - exercises some core `Central` CR features. - [central/central-misc](central/central-misc) - various `Central` resource configurations and switching between them. - [securedcluster/sc-basic](securedcluster/sc-basic) - various `SecuredCluster` resource configurations @@ -23,7 +24,7 @@ Note that the `deploy-via-olm` and `deploy-previous-via-olm` targets in `Makefil In a CI job: 1. a previous operator version is installed first (a requirement of the upgrade test) using the `deploy-previous-via-olm` rule, 2. the upgrade test runs next; it leaves the cluster with current operator version installed, -3. then the `central` and `securedcluster` test suites are run _in parallel_. +3. then the `controller`, `central` and `securedcluster` test suites are run _in parallel_. An important fact is that `kuttl` creates a uniquely-named ephemeral namespace (something like `kuttl-test-adjective-animal`) for every test, and deletes it afterwards. diff --git a/operator/tests/controller/metrics/100-assert-rbac-works.yaml b/operator/tests/controller/metrics/100-assert-rbac-works.yaml new file mode 100644 index 0000000000000..7012daac075ad --- /dev/null +++ b/operator/tests/controller/metrics/100-assert-rbac-works.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- command: kubectl auth can-i get /metrics --as=system:serviceaccount:$NAMESPACE:operator-metrics-privileged --quiet diff --git a/operator/tests/controller/metrics/100-rbac.yaml b/operator/tests/controller/metrics/100-rbac.yaml new file mode 100644 index 0000000000000..eee44f980e68a --- /dev/null +++ b/operator/tests/controller/metrics/100-rbac.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator-metrics-unprivileged +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator-metrics-privileged +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +# We cannot simply apply a ClusterRoleBinding resource, because of the variable $NAMESPACE +- command: kubectl create clusterrolebinding rhacs-metrics-$NAMESPACE --clusterrole=rhacs-operator-metrics-reader --serviceaccount=$NAMESPACE:operator-metrics-privileged diff --git a/operator/tests/controller/metrics/200-access-no-auth.yaml b/operator/tests/controller/metrics/200-access-no-auth.yaml new file mode 100644 index 0000000000000..943f05d702a75 --- /dev/null +++ b/operator/tests/controller/metrics/200-access-no-auth.yaml @@ -0,0 +1,43 @@ +# Ensure that unauthenticated requests are refused with a 401. +# This pod definition should match its siblings, except for the name, serviceAccount and command. +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-no-auth + labels: + test: metrics-access +spec: + restartPolicy: Never + containers: + - name: run + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + args: + - bash + - "-c" + # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported + - >- + operator_ns="stackrox-operator" + url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics"; + set -u; + curl --version; + for attempt in $(seq 5); do + echo Attempt $attempt:; + curl --insecure --dump-header /tmp/headers "$url"; + echo response:; + head -n 1 /tmp/headers; + echo checking response headers:; + if grep -Ei "^HTTP/[0-9.]+ 401" /tmp/headers; then exit 0; fi; + sleep 1; + done; + exit 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + serviceAccount: operator-metrics-unprivileged + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault diff --git a/operator/tests/controller/metrics/200-access-no-http2.yaml b/operator/tests/controller/metrics/200-access-no-http2.yaml new file mode 100644 index 0000000000000..ac6b0be21de97 --- /dev/null +++ b/operator/tests/controller/metrics/200-access-no-http2.yaml @@ -0,0 +1,43 @@ +# Ensure that the metrics server correctly refuses to talk HTTP/2 (Rapid Reset mitigation). +# This pod definition should match its siblings, except for the name, serviceAccount and command. +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-no-http2 + labels: + test: metrics-access +spec: + restartPolicy: Never + containers: + - name: run + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + args: + - bash + - "-c" + # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported + - >- + operator_ns="stackrox-operator" + url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics"; + set -u; + curl --version; + for attempt in $(seq 5); do + echo Attempt $attempt:; + curl --http2 --insecure --dump-header /tmp/headers "$url"; + echo response:; + head -n 1 /tmp/headers; + echo checking response headers:; + if grep -Ei "^HTTP/1" /tmp/headers; then exit 0; fi; + sleep 1; + done; + exit 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + serviceAccount: operator-metrics-unprivileged + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault diff --git a/operator/tests/controller/metrics/200-access-privileged.yaml b/operator/tests/controller/metrics/200-access-privileged.yaml new file mode 100644 index 0000000000000..98035c9fdbba7 --- /dev/null +++ b/operator/tests/controller/metrics/200-access-privileged.yaml @@ -0,0 +1,44 @@ +# Ensure that authenticated requests from service account with necessary permission are accepted and return metrics. +# This pod definition should match its siblings, except for the name, serviceAccount and command. +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-privileged + labels: + test: metrics-access +spec: + restartPolicy: Never + containers: + - name: run + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + args: + - bash + - "-c" + # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported + - >- + operator_ns="stackrox-operator" + url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics"; + set -eu; + curl --version; + token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"; + for attempt in $(seq 5); do + echo Attempt $attempt:; + curl --insecure --fail -H "Authorization: Bearer $token" "$url" > /tmp/response; + echo beginning of response body:; + head /tmp/response; + echo checking response body:; + if grep -Ei "^# TYPE " /tmp/response; then exit 0; fi; + sleep 1; + done; + exit 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + serviceAccount: operator-metrics-privileged + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault diff --git a/operator/tests/controller/metrics/200-access-unprivileged.yaml b/operator/tests/controller/metrics/200-access-unprivileged.yaml new file mode 100644 index 0000000000000..1d4576ccd99a6 --- /dev/null +++ b/operator/tests/controller/metrics/200-access-unprivileged.yaml @@ -0,0 +1,44 @@ +# Ensure that authenticated requests from service account without necessary permissions are refused with a 403. +# This pod definition should match its siblings, except for the name, serviceAccount and command. +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-unprivileged + labels: + test: metrics-access +spec: + restartPolicy: Never + containers: + - name: run + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + args: + - bash + - "-c" + # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported + - >- + operator_ns="stackrox-operator" + url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics"; + set -u; + curl --version; + token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"; + for attempt in $(seq 5); do + echo Attempt $attempt:; + curl --insecure --dump-header /tmp/headers -H "Authorization: Bearer $token" "$url" > /tmp/response; + echo response:; + head -n 1 /tmp/headers; + echo checking response headers:; + if grep -Ei "^HTTP/[0-9.]+ 403" /tmp/headers; then exit 0; fi; + sleep 1; + done; + exit 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + serviceAccount: operator-metrics-unprivileged + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault diff --git a/operator/tests/controller/metrics/200-assert.yaml b/operator/tests/controller/metrics/200-assert.yaml new file mode 100644 index 0000000000000..15f5ad554e009 --- /dev/null +++ b/operator/tests/controller/metrics/200-assert.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-no-auth +status: + phase: Succeeded +--- +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-no-http2 +status: + phase: Succeeded +--- +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-privileged +status: + phase: Succeeded +--- +apiVersion: v1 +kind: Pod +metadata: + name: operator-metrics-unprivileged +status: + phase: Succeeded +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +collectors: +- type: pod + selector: test=metrics-access + tail: -1 +- command: kubectl describe pod -n $NAMESPACE -l test=metrics-access