Make the kubeflow-m2m-oidc-configurator a CronJob (#2667)

* Make the kubeflow-m2m-oidc-configurator a CronJob Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski.kr3@roche.com> Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * cronjob.kubeflow-m2m-oidc-configurator: concurrencyPolicy: Forbid Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski.kr3@roche.com> Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * add tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * Improve wait routine for m2m oidc configurator (#2) It was tested with self-hosted runner using custom dockerconfig credentials for debugging. Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * use docker.io/curlimages/curl for cronjob.kubeflow-m2m-oidc-configurator.yaml Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * make the m2m oidc configurator idempotent Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> * verify jwks configuration in requestauthentication Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com> --------- Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski.kr3@roche.com> Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>
kubeflow · Jun 21, 2024 · a1dbf47 · a1dbf47
1 parent 81ed256
commit a1dbf47
Show file tree

Hide file tree

Showing 14 changed files with 235 additions and 99 deletions.
diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml
@@ -39,7 +39,7 @@ jobs:
         run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
       - name: Install Istio with ext auth
-        run: ./tests/gh-actions/install_istio_with_ext_auth.sh*
+        run: ./tests/gh-actions/install_istio_with_ext_auth.sh
 
       - name: Install cert-manager
         run: ./tests/gh-actions/install_cert_manager.sh
@@ -67,6 +67,10 @@ jobs:
           nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
           while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
+      - name: Wait for the kubeflow-m2m-oidc-configurator Job
+        run: |
+          ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
+
       - name: Run kserve tests with m2m token from SA default/default
         run: |
           export KSERVE_INGRESS_HOST_PORT=localhost:8080

diff --git a/.github/workflows/notebook_controller_m2m_test.yaml b/.github/workflows/notebook_controller_m2m_test.yaml
@@ -35,7 +35,7 @@ jobs:
       run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
     - name: Install Istio with ext auth
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh*
+      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
 
     - name: Install kubeflow-istio-resources
       run: kustomize build common/istio-1-22/kubeflow-istio-resources/base | kubectl apply -f -
@@ -47,7 +47,8 @@ jobs:
       run: |
         kustomize build apps/jupyter/jupyter-web-app/upstream/overlays/istio/ | kubectl apply -f -
         kustomize build apps/jupyter/notebook-controller/upstream/overlays/kubeflow/ | kubectl apply -f -
-        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 300s
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s \
+          --field-selector=status.phase!=Succeeded
 
     - name: Create KF Profile
       run: kustomize build common/user-namespace/base | kubectl apply -f -
@@ -58,6 +59,10 @@ jobs:
         nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
         while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
+    - name: Wait for the kubeflow-m2m-oidc-configurator Job
+      run: |
+        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
+
     - name: List notebooks over API with authorized SA Token
       run: |
         KF_PROFILE=kubeflow-user-example-com
@@ -86,4 +91,4 @@ jobs:
         if test $STATUS_CODE -ne 403; then
           echo "Error, this call should fail to list notebooks in namespace ${KF_PROFILE}."
           exit 1
-        fi
+        fi
diff --git a/.github/workflows/pipeline_test.yaml b/.github/workflows/pipeline_test.yaml
@@ -60,6 +60,10 @@ jobs:
           nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
           while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
+      - name: Wait for the kubeflow-m2m-oidc-configurator Job
+        run: |
+          ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
+
       - name: List and deploy test pipeline with authorized ServiceAccount Token
         run: |
           pip3 install kfp==2.4.0
@@ -116,4 +120,4 @@ jobs:
           ' "${TOKEN}" "${KF_PROFILE}"
 
           echo "Test succeeded. Token from unauthorized ServiceAccount cannot list \
-          piplines in $KF_PROFILE namespace."
+          piplines in $KF_PROFILE namespace."
diff --git a/.../configure-self-signed-kubernetes-oidc-issuer/cronjob.kubeflow-m2m-oidc-configurator.yaml b/.../configure-self-signed-kubernetes-oidc-issuer/cronjob.kubeflow-m2m-oidc-configurator.yaml
@@ -0,0 +1,39 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: kubeflow-m2m-oidc-configurator
+  namespace: istio-system
+spec:
+  schedule: '*/5 * * * *'
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 3
+      ttlSecondsAfterFinished: 600
+      template:
+        metadata:
+          labels: {}
+        spec:
+          restartPolicy: OnFailure
+          serviceAccountName: kubeflow-m2m-oidc-configurator
+          containers:
+            - image: docker.io/curlimages/curl
+              name: kubeflow-m2m-oidc-configurator
+              command:
+                - /script.sh
+              envFrom:
+                - configMapRef:
+                    name: kubeflow-m2m-oidc-configurator-envs
+              volumeMounts:
+                - mountPath: /script.sh
+                  name: script
+                  subPath: script.sh
+              resources: {}
+          volumes:
+            - name: script
+              configMap:
+                name: kubeflow-m2m-oidc-configurator-script
+                defaultMode: 0777
+                items:
+                  - key: script.sh
+                    path: script.sh
diff --git a/...netes-oidc-issuer/job.configure-kubernetes-oidc-issuer-jwks-in-requestauthentication.yaml b/...netes-oidc-issuer/job.configure-kubernetes-oidc-issuer-jwks-in-requestauthentication.yaml
diff --git a/...t/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/kustomization.yaml b/...t/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/kustomization.yaml
@@ -2,16 +2,16 @@ apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 
 resources:
-- job.configure-kubernetes-oidc-issuer-jwks-in-requestauthentication.yaml
+- cronjob.kubeflow-m2m-oidc-configurator.yaml
 - rbac.yaml
 
 configMapGenerator:
-- name: configure-self-signed-kubernetes-oidc-issuer-script
+- name: kubeflow-m2m-oidc-configurator-script
   namespace: istio-system
   files:
   - script.sh=script.sh
 
-- name: configure-self-signed-kubernetes-oidc-issuer-envs
+- name: kubeflow-m2m-oidc-configurator-envs
   namespace: istio-system
   literals:
   - ISTIO_ROOT_NAMESPACE=istio-system

diff --git a/...idc-client/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/rbac.yaml b/...idc-client/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/rbac.yaml
@@ -1,14 +1,14 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: self-signed-kubernetes-oidc-issuer-configurator
+  name: kubeflow-m2m-oidc-configurator
   namespace: istio-system
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: self-signed-kubernetes-oidc-issuer-configurator
+  name: kubeflow-m2m-oidc-configurator
   namespace: istio-system
 rules:
   - apiGroups:
@@ -23,13 +23,13 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: self-signed-kubernetes-oidc-issuer-configurator
+  name: kubeflow-m2m-oidc-configurator
   namespace: istio-system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: self-signed-kubernetes-oidc-issuer-configurator
+  name: kubeflow-m2m-oidc-configurator
 subjects:
   - kind: ServiceAccount
-    name: self-signed-kubernetes-oidc-issuer-configurator
+    name: kubeflow-m2m-oidc-configurator
     namespace: istio-system
diff --git a/...idc-client/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/script.sh b/...idc-client/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/script.sh
@@ -12,60 +12,132 @@ ${ISTIO_ROOT_NAMESPACE}\
 /requestauthentications/\
 ${REQUEST_AUTHENTICATION_NAME}"
 
-echo "Wait until resource RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} in namespace ${ISTIO_ROOT_NAMESPACE} is ready."
-while true; do
-  response="$(
-    curl -s -o /dev/null \
-      --url "${RESOURCE_URL}" \
-      -w "%{http_code}" \
-      --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
-      --insecure
-  )"
-  if [ "${response}" = "200" ]; then
-    break
-  fi
-  echo "Resource RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} in namespace ${ISTIO_ROOT_NAMESPACE} is not ready yet."
-  sleep 5
-done
-echo "Resource RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} in namespace ${ISTIO_ROOT_NAMESPACE} is ready."
+wait_for_resource_ready() {
+  while true; do
+    response="$(
+      curl -s -o /dev/null \
+        --url "${RESOURCE_URL}" \
+        -w "%{http_code}" \
+        --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
+        --insecure
+    )"
+    if [ "${response}" = "200" ]; then
+      break
+    fi
+    sleep 5
+  done
+}
 
-# Get Issuer URL configured in RequestAuthentication.
-ISSUER_URL="$(
+get_request_authentication_obj() {
   curl -s --request GET \
     --url "${RESOURCE_URL}" \
     --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
-    --insecure |
-    awk -F'"' '/"issuer":/ { print $4 }'
-)"
-echo "ISSUER_URL: ${ISSUER_URL}."
+    --insecure
+}
+
+get_issuer_url_from_obj() {
+  obj="${1}"
+  echo "${obj}" | awk -F'"' '/"issuer":/ { print $4 }'
+}
 
-# GET URI to the JWKS.
-JWKS_URI="$(
+get_current_escaped_jwks_from_obj() {
+  obj="${1}"
+  echo "${obj}" | awk -F'"' '/"jwks":/' | sed -n 's/^.*"jwks": "\(.*\)".*$/\1/p'
+}
+
+get_jwks_uri() {
+  issuer_url="${1}"
   curl -s --request GET \
-    --url "${ISSUER_URL}/.well-known/openid-configuration" \
-    --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
-    --insecure |
-    grep -o '"jwks_uri":"https:\/\/[^"]\+"' |
-    sed 's/"jwks_uri":"\(.*\)"/\1/'
-)"
-echo "JWKS_URI: ${JWKS_URI}."
-
-# Get content of the JWKS.
-JWKS="$(
+      --url "${issuer_url}/.well-known/openid-configuration" \
+      --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
+      --insecure |
+      grep -o '"jwks_uri":"https:\/\/[^"]\+"' |
+      sed 's/"jwks_uri":"\(.*\)"/\1/'
+  }
+
+get_jwks_from_uri() {
+  jwks_uri="${1}"
   curl -s --request GET \
-    --url "${JWKS_URI}" \
+      --url "${jwks_uri}" \
+      --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
+      --insecure
+}
+
+# Format JWKS in a way that can be accepted in resource patch.
+parse_escaped_jwks() {
+  jwks="${1}"
+  echo "${jwks}" | sed 's/"/\\"/g'
+}
+
+are_jwks_equal() {
+  jwks1="${1}"
+  jwks2="${2}"
+  test "$(echo "${jwks1}" | base64 -w0)" = "$(echo "${jwks2}" | base64 -w0)"
+}
+
+patch_request_authentication_with_escaped_jwks() {
+  jwks_escaped="${1}"
+  curl -s --request PATCH \
+    --url "${RESOURCE_URL}" \
+    --header "Content-Type: application/json-patch+json" \
     --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
+    -d '[{ "op": "add", "path": "/spec/jwtRules/0/jwks", "value": "'"${jwks_escaped}"'" }]' \
     --insecure
-)"
-echo "JWKS: ${JWKS}."
+    echo
+}
 
-# Format JWKS in a way that can be accepted in resource patch.
-JWKS_ESCAPED="$(echo "${JWKS}" | sed 's/"/\\"/g')"
-
-# Patch the RequestAuthentication with JWKS.
-curl -s --request PATCH \
-  --url "${RESOURCE_URL}" \
-  --header "Content-Type: application/json-patch+json" \
-  --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" \
-  -d '[{ "op": "add", "path": "/spec/jwtRules/0/jwks", "value": "'"${JWKS_ESCAPED}"'" }]' \
-  --insecure
+patch_request_authentication_with_jwks_if_required() {
+  echo "Getting RequestAuthentication object."
+  REQUEST_AUTHENTICATION_OBJ="$(get_request_authentication_obj)"
+
+  ISSUER_URL="$(get_issuer_url_from_obj "${REQUEST_AUTHENTICATION_OBJ}")"
+  echo "Issuer Url in RequestAuthentication: ${ISSUER_URL}"
+
+  CURRENT_JWKS_ESCAPED="$(get_current_escaped_jwks_from_obj "${REQUEST_AUTHENTICATION_OBJ}")"
+  printf "Current Jwks (escaped):\n%s\n" "${CURRENT_JWKS_ESCAPED}"
+
+  JWKS_URI="$(get_jwks_uri "${ISSUER_URL}")"
+  echo "Jwks Uri from Well Known OpenID Configuration: ${JWKS_URI}"
+
+  JWKS="$(get_jwks_from_uri "${JWKS_URI}")"
+  JWKS_ESCAPED="$(parse_escaped_jwks "${JWKS}")"
+  printf "JWKS from Well Known OpenID Configuration (escaped): \n%s\n" "${JWKS_ESCAPED}"
+
+  if are_jwks_equal "${JWKS_ESCAPED}" "${CURRENT_JWKS_ESCAPED}"; then
+    echo "JWKS in RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} is configured correctly."
+  else
+    echo "JWKS in RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} needs to be configured."
+    patch_request_authentication_with_escaped_jwks "${JWKS_ESCAPED}"
+  fi
+}
+
+verify_jwks_in_request_authentication() {
+  REQUEST_AUTHENTICATION_OBJ="$(get_request_authentication_obj)"
+  ISSUER_URL="$(get_issuer_url_from_obj "${REQUEST_AUTHENTICATION_OBJ}")"
+  CURRENT_JWKS_ESCAPED="$(get_current_escaped_jwks_from_obj "${REQUEST_AUTHENTICATION_OBJ}")"
+  JWKS_URI="$(get_jwks_uri "${ISSUER_URL}")"
+  JWKS="$(get_jwks_from_uri "${JWKS_URI}")"
+  JWKS_ESCAPED="$(parse_escaped_jwks "${JWKS}")"
+  if ! are_jwks_equal "${JWKS_ESCAPED}" "${CURRENT_JWKS_ESCAPED}"; then
+    echo "JWKS not properly configured, exit with error code 1"
+    exit 1
+  fi
+}
+
+main() {
+  echo "Wait until resource RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} in namespace ${ISTIO_ROOT_NAMESPACE} is ready."
+  wait_for_resource_ready
+  echo "Resource RequestAuthentication ${REQUEST_AUTHENTICATION_NAME} in namespace ${ISTIO_ROOT_NAMESPACE} is ready."
+
+  echo "Patch RequestAuthentication with JWKS if required."
+  patch_request_authentication_with_jwks_if_required
+
+  echo "Wait 5 seconds before verifying RequestAuthentication JWKS configuration."
+  sleep 5
+
+  echo "Verify if RequestAuthentication is properly configured with JWKS..."
+  verify_jwks_in_request_authentication
+  echo "RequestAuthentication is properly configured with JWKS."
+}
+
+main
diff --git a/tests/gh-actions/install_istio_with_ext_auth.sh b/tests/gh-actions/install_istio_with_ext_auth.sh
@@ -8,7 +8,8 @@ kustomize build istio-install/overlays/oauth2-proxy | kubectl apply -f -
 cd -
 
 echo "Waiting for all Istio Pods to become ready..."
-kubectl wait --for=condition=Ready pods --all -n istio-system --timeout 300s
+kubectl wait --for=condition=Ready pods --all -n istio-system --timeout=300s \
+  --field-selector=status.phase!=Succeeded
 
 echo "Installing oauth2-proxy..."
 cd common/oidc-client