fix: resolve install failure in CN setup (#7439)

* fix: handle upgrade issues Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: fix incorrect ingress and labels Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: raise error if KC db password cannot be resolved Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: avoid race condition while configuring keycloak Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: add missing keys in jans-lock config Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: remove invocation of updatelbip.py script Signed-off-by: iromli <isman.firmansyah@gmail.com> * ci: optimize nginx (#7440) * ci: optimize proxy settings Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> * ci: optimize rdbms settings Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> * ci: fix version Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> --------- Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> * chore: change ARG CN_VERSION to BASE_VERSION for linking base images Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: add missing /opt/jans/bin directory Signed-off-by: iromli <isman.firmansyah@gmail.com> * chore: sync templates from upstream jans-linux-setup Signed-off-by: iromli <isman.firmansyah@gmail.com> * revert: rdbms properties Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> --------- Signed-off-by: iromli <isman.firmansyah@gmail.com> Signed-off-by: moabu <47318409+moabu@users.noreply.github.com> Co-authored-by: Mohammad Abudayyeh <47318409+moabu@users.noreply.github.com>
JanssenProject · Jan 22, 2024 · f84e99a · f84e99a
1 parent b88bab6
commit f84e99a
Show file tree

Hide file tree

Showing 33 changed files with 162 additions and 80 deletions.
diff --git a/.github/workflows/docker_build_image.yml b/.github/workflows/docker_build_image.yml
@@ -134,17 +134,18 @@ jobs:
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
           echo "build=${BUILD}" >> $GITHUB_OUTPUT
           echo "dockerfilelocation=${DOCKER_FILE_LOCATION}" >> $GITHUB_OUTPUT
-          
-          # wait for all images in DEFAULT_ALL to be built before building the all-in-one image as it depends on all other images
-          if [[ "docker-jans-all-in-one" =~ "${{ matrix.docker-images }}" ]]; then
-            TEMP_IMG="auth-server certmanager config-api configurator fido2 persistence-loader scim monolith loadtesting-jmeter link casa saml keycloak-link"
-            for i in $TEMP_IMG; do
-              TEMP_TOKEN=$(curl https://ghcr.io/token\?scope\="repository:janssenproject/jans/$i:pull" | jq -r '.token')
-              while [[ $(curl -s -H "Authorization: Bearer ${TEMP_TOKEN}" https://ghcr.io/v2/janssenproject/jans/$i/tags/list | jq -r '.tags' | grep "$VERSION" | tr -d '[:space:]"') != "$VERSION" ]]; do
-                echo "Waiting for $i to be built"
-                sleep 30
+          if [[ ! $VERSION =~ "_dev" ]]; then
+            # wait for all images in DEFAULT_ALL to be built before building the all-in-one image as it depends on all other images
+            if [[ "docker-jans-all-in-one" =~ "${{ matrix.docker-images }}" ]]; then
+              TEMP_IMG="auth-server certmanager config-api configurator fido2 persistence-loader scim monolith loadtesting-jmeter link casa saml keycloak-link"
+              for i in $TEMP_IMG; do
+                TEMP_TOKEN=$(curl https://ghcr.io/token\?scope\="repository:janssenproject/jans/$i:pull" | jq -r '.token')
+                while [[ $(curl -s -H "Authorization: Bearer ${TEMP_TOKEN}" https://ghcr.io/v2/janssenproject/jans/$i/tags/list | jq -r '.tags' | grep "$VERSION" | tr -d '[:space:]"') != "$VERSION" ]]; do
+                  echo "Waiting for $i to be built"
+                  sleep 30
+                done
               done
-            done
+            fi
           fi
 
       # UPDATE BUILD DATES INSIDE THE DOCKERFILE BEFORE BUILDING THE DEV IMAGES TRIGGERED BY JENKINS

diff --git a/charts/janssen-all-in-one/templates/deployment.yml b/charts/janssen-all-in-one/templates/deployment.yml
@@ -58,9 +58,6 @@ spec:
               {{- with .Values.customScripts }}
               {{- toYaml . | replace "- " "" | nindent 14}}
               {{- end }}
-              {{- if not .Values.isFqdnRegistered }}
-              /usr/bin/python3 /scripts/updatelbip.py &
-              {{- end}}
               /app/bin/entrypoint.sh
         {{- end}}
         ports:

diff --git a/charts/janssen-all-in-one/templates/nginx-ingress.yaml b/charts/janssen-all-in-one/templates/nginx-ingress.yaml
@@ -612,7 +612,6 @@ spec:
                 port:
                   number: 8080
 {{- end }}
-{{- end }}
 
 ---
 

diff --git a/charts/janssen-all-in-one/templates/secret.yaml b/charts/janssen-all-in-one/templates/secret.yaml
@@ -155,7 +155,7 @@ kind: Secret
 metadata:
   name: {{ .Release.Name }}-kc-db-pass
   labels:
-{{ include "config.labels" . | indent 4 }}
+{{ include "janssen-all-in-one.labels" . | indent 4 }}
 {{- if .Values.additionalLabels }}
 {{ toYaml .Values.additionalLabels | indent 4 }}
 {{- end }}

diff --git a/docker-jans-all-in-one/Dockerfile b/docker-jans-all-in-one/Dockerfile
@@ -2,21 +2,21 @@
 # Assets sources
 # ==============
 
-# original Janssen version
-ARG CN_VERSION=1.0.22
+# original Janssen base version (without suffix)
+ARG BASE_VERSION=1.0.22
 
 # the following ARGs set default base images
 # they can be overriden in build process via --build-arg option
-ARG JANS_CONFIGURATOR_IMAGE=ghcr.io/janssenproject/jans/configurator:${CN_VERSION}_dev
-ARG JANS_PERSISTENCE_LOADER_IMAGE=ghcr.io/janssenproject/jans/persistence-loader:${CN_VERSION}_dev
-ARG JANS_AUTH_IMAGE=ghcr.io/janssenproject/jans/auth-server:${CN_VERSION}_dev
-ARG JANS_CONFIG_API_IMAGE=ghcr.io/janssenproject/jans/config-api:${CN_VERSION}_dev
-ARG JANS_FIDO2_IMAGE=ghcr.io/janssenproject/jans/fido2:${CN_VERSION}_dev
-ARG JANS_SCIM_IMAGE=ghcr.io/janssenproject/jans/scim:${CN_VERSION}_dev
-ARG JANS_CASA_IMAGE=ghcr.io/janssenproject/jans/casa:${CN_VERSION}_dev
-ARG JANS_LINK_IMAGE=ghcr.io/janssenproject/jans/link:${CN_VERSION}_dev
-ARG JANS_KEYCLOAK_LINK_IMAGE=ghcr.io/janssenproject/jans/keycloak-link:${CN_VERSION}_dev
-ARG JANS_SAML_IMAGE=ghcr.io/janssenproject/jans/saml:${CN_VERSION}_dev
+ARG JANS_CONFIGURATOR_IMAGE=ghcr.io/janssenproject/jans/configurator:${BASE_VERSION}_dev
+ARG JANS_PERSISTENCE_LOADER_IMAGE=ghcr.io/janssenproject/jans/persistence-loader:${BASE_VERSION}_dev
+ARG JANS_AUTH_IMAGE=ghcr.io/janssenproject/jans/auth-server:${BASE_VERSION}_dev
+ARG JANS_CONFIG_API_IMAGE=ghcr.io/janssenproject/jans/config-api:${BASE_VERSION}_dev
+ARG JANS_FIDO2_IMAGE=ghcr.io/janssenproject/jans/fido2:${BASE_VERSION}_dev
+ARG JANS_SCIM_IMAGE=ghcr.io/janssenproject/jans/scim:${BASE_VERSION}_dev
+ARG JANS_CASA_IMAGE=ghcr.io/janssenproject/jans/casa:${BASE_VERSION}_dev
+ARG JANS_LINK_IMAGE=ghcr.io/janssenproject/jans/link:${BASE_VERSION}_dev
+ARG JANS_KEYCLOAK_LINK_IMAGE=ghcr.io/janssenproject/jans/keycloak-link:${BASE_VERSION}_dev
+ARG JANS_SAML_IMAGE=ghcr.io/janssenproject/jans/saml:${BASE_VERSION}_dev
 
 # -----------
 # base images
@@ -109,6 +109,7 @@ COPY --from=jans-config-api-src /app/templates/jans-config-api /app/templates/ja
 COPY --from=jans-config-api-src /usr/bin/facter /usr/local/bin/facter
 COPY --from=jans-config-api-src /app/scripts /app/jans_aio/jans_config_api
 COPY --from=jans-config-api-src /var/log/adminui /var/log/adminui
+COPY --from=jans-config-api-src /opt/jans/bin /opt/jans/bin
 RUN ln -sf /app/jans_aio/jans_config_api/entrypoint.sh /app/bin/jans-config-api-entrypoint.sh
 
 COPY --from=jans-fido2-src /opt/jans/jetty/jans-fido2 /opt/jans/jetty/jans-fido2
@@ -159,6 +160,9 @@ RUN mkdir -p /opt/jans/configurator/db \
 
 COPY app /app
 
+# CN version as env var (with suffix if any, i.e. SNAPSHOT)
+ENV CN_VERSION=1.0.22-SNAPSHOT
+
 # set directory contains installer code that will be added to Python sys.path
 ENV PYTHONPATH=/app
 

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-auth-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-auth-location.conf
@@ -13,7 +13,11 @@ location /.well-known/openid-configuration {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -34,7 +38,11 @@ location /.well-known/webfinger {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -55,7 +63,11 @@ location /.well-known/uma2-configuration {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -76,7 +88,11 @@ location /firebase-messaging-sw.js {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -97,7 +113,11 @@ location /device-code {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -118,7 +138,11 @@ location /jans-auth {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-auth-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-auth-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_auth_backend {
     server 127.0.0.1:8081;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-casa-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-casa-location.conf
@@ -13,7 +13,11 @@ location /jans-casa {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-casa-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-casa-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_casa_backend {
     server 127.0.0.1:8082;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-config-api-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-config-api-location.conf
@@ -13,7 +13,11 @@ location /jans-config-api {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-config-api-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-config-api-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_config_api_backend {
     server 127.0.0.1:8074;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-fido2-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-fido2-location.conf
@@ -13,7 +13,11 @@ location /.well-known/fido2-configuration {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -34,7 +38,11 @@ location /jans-fido2 {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-fido2-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-fido2-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_fido2_backend {
     server 127.0.0.1:8073;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-keycloak-link-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-keycloak-link-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_keycloak_link_backend {
     server 127.0.0.1:9092;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-link-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-link-location.conf
@@ -13,7 +13,10 @@ location /jans-link {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-link-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-link-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_link_backend {
     server 127.0.0.1:9091;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-saml-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-saml-location.conf
@@ -13,7 +13,10 @@ location /kc {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-saml-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-saml-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_saml_backend {
     server 127.0.0.1:8083;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-scim-location.conf b/docker-jans-all-in-one/app/templates/nginx/jans-scim-location.conf
@@ -13,7 +13,11 @@ location /.well-known/scim-configuration {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;
@@ -34,7 +38,11 @@ location /jans-scim {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-    proxy_connect_timeout 5s;
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+
     proxy_redirect off;
     port_in_redirect off;
     proxy_http_version 1.1;

diff --git a/docker-jans-all-in-one/app/templates/nginx/jans-scim-upstream.conf b/docker-jans-all-in-one/app/templates/nginx/jans-scim-upstream.conf
@@ -1,4 +1,3 @@
 upstream jans_scim_backend {
     server 127.0.0.1:8087;
-    keepalive 2;
 }
diff --git a/docker-jans-all-in-one/app/templates/nginx/nginx-default.conf b/docker-jans-all-in-one/app/templates/nginx/nginx-default.conf
@@ -5,9 +5,13 @@ geo $literal_dollar {
 
 %(upstream_includes)s
 
+client_header_timeout 300;
+client_body_timeout 300;
+
 server {
     listen 8080 default_server;
     server_name _;
+    gzip on;
 
     root /var/lib/nginx/html;
     index index.html;

diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile
@@ -101,7 +101,7 @@ RUN mkdir -p ${JETTY_BASE}/jans-auth/agama/fl \
     /app/static/rdbm \
     /app/schema
 
-ENV JANS_SOURCE_VERSION=0479535f2f33890ffcb0bd6589eb8ebbd950ce96
+ENV JANS_SOURCE_VERSION=647aa0b0a118ab629e185b7f67ab3ded87ea1b9c
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 
 # note that as we're pulling from a monorepo (with multiple project in it)

diff --git a/docker-jans-auth-server/scripts/upgrade.py b/docker-jans-auth-server/scripts/upgrade.py
@@ -25,12 +25,31 @@
 
 def _transform_lock_dynamic_config(conf):
     should_update = False
+
     opa_url = os.environ.get("CN_OPA_URL", "http://localhost:8181/v1")
 
     if opa_url != conf["opaConfiguration"]["baseUrl"]:
         conf["opaConfiguration"]["baseUrl"] = opa_url
         should_update = True
 
+    # add missing top-level keys
+    for missing_key, value in [
+        ("policiesJsonUrisAccessToken", ""),
+        ("policiesZipUris", []),
+        ("policiesZipUrisAccessToken", ""),
+    ]:
+        if missing_key not in conf:
+            conf[missing_key] = value
+            should_update = True
+
+    # add missing opaConfiguration-level keys
+    for missing_key, value in [
+        ("accessToken", ""),
+    ]:
+        if missing_key not in conf["opaConfiguration"]:
+            conf["opaConfiguration"][missing_key] = value
+            should_update = True
+
     # return modified config (if any) and update flag
     return conf, should_update