feat(image): generate scopes from swagger/openapi files (#2759)

* refactor(image): persistence entry insertion of config-api

Ref: issue #2724

* refactor(image): remove dead code after moving config-api scopes insertion

* refactor(image): generate config-api scopes from swagger file

* feat(image): create SCIM scopes from swagger/openapi file

Ref: #2744

* chore(jans-pycloudlib): updated build (#2760)

Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com>

Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com>

* refactor(image): defer scim-plugin scopes creation when running config-api

Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com>
Co-authored-by: mo-auto <54212639+mo-auto@users.noreply.github.com>

docker-jans-auth-server/requirements.txt

@@ -1,4 +1,4 @@
 # pinned to py3-grpcio version to avoid failure on native extension build
 grpcio==1.41.0
 libcst<0.4
-git+https://github.com/JanssenProject/jans@9b536ab2b5d398a41733790f2eeb70339f993fb7#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans@42f2834680de9578bce5f39ac5de35de19a6c48d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-certmanager/requirements.txt

@@ -2,4 +2,4 @@
 grpcio==1.41.0
 click==6.7
 libcst<0.4
-git+https://github.com/JanssenProject/jans@9b536ab2b5d398a41733790f2eeb70339f993fb7#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans@42f2834680de9578bce5f39ac5de35de19a6c48d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-config-api/.dockerignore

@@ -9,3 +9,4 @@
 !scripts
 !LICENSE
 !requirements.txt
+!templates

docker-jans-config-api/.hadolint.yaml

@@ -2,3 +2,4 @@ ignored:
   - DL3018    # Pin versions in apk add
   - DL3013    # Pin versions in pip
   - DL3003    # Use WORKDIR to switch to a directory
+  - SC2016

docker-jans-config-api/Dockerfile

@@ -27,12 +27,23 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
     && mv /opt/jetty-home-${JETTY_VERSION} ${JETTY_HOME} \
     && rm -rf /tmp/jetty.tar.gz
 
+# ======
+# Jython
+# ======
+
+ARG JYTHON_VERSION=2.7.3
+ARG JYTHON_BUILD_DATE='2022-08-01 07:49'
+RUN wget -q https://maven.jans.io/maven/io/jans/jython-installer/${JYTHON_VERSION}/jython-installer-${JYTHON_VERSION}.jar -O /tmp/jython-installer.jar \
+    && mkdir -p /opt/jython \
+    && java -jar /tmp/jython-installer.jar -v -s -d /opt/jython \
+    && rm -f /tmp/jython-installer.jar /tmp/*.properties
+
 # ==========
 # Config API
 # ==========
 
 ENV CN_VERSION=1.0.3-SNAPSHOT
-ENV CN_BUILD_DATE='2022-10-14 16:35'
+ENV CN_BUILD_DATE='2022-10-24 13:36'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API
@@ -99,6 +110,41 @@ RUN mkdir -p /opt/prometheus \
     && wget -q https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${PROMETHEUS_JAVAAGENT_VERSION}/jmx_prometheus_javaagent-${PROMETHEUS_JAVAAGENT_VERSION}.jar -O /opt/prometheus/jmx_prometheus_javaagent.jar \
     && java -jar ${JETTY_HOME}/start.jar jetty.home=${JETTY_HOME} jetty.base=${JETTY_BASE}/jans-config-api --add-module=jmx,stats
 
+# =====================
+# jans-linux-setup sync
+# =====================
+
+ENV JANS_SOURCE_VERSION=e74ea8e27e59d35ff6e3c6f997e6c1df6a04ec83
+ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
+ARG JANS_CONFIG_API_DOCS=jans-config-api/docs
+
+# note that as we're pulling from a monorepo (with multiple project in it)
+# we are using partial-clone and sparse-checkout to get the jans-linux-setup code
+RUN git clone --filter blob:none --no-checkout https://github.com/janssenproject/jans /tmp/jans \
+    && cd /tmp/jans \
+    && git sparse-checkout init --cone \
+    && git checkout ${JANS_SOURCE_VERSION} \
+    && git sparse-checkout add ${JANS_SETUP_DIR} \
+    && git sparse-checkout add ${JANS_CONFIG_API_DOCS}
+
+RUN mkdir -p /etc/jans/conf \
+    /app/static/rdbm \
+    /app/schema \
+    /app/templates/jans-config-api
+
+# sync static files from linux-setup
+RUN cd /tmp/jans \
+    && cp ${JANS_SETUP_DIR}/static/rdbm/sql_data_types.json /app/static/rdbm/ \
+    && cp ${JANS_SETUP_DIR}/static/rdbm/ldap_sql_data_type_mapping.json /app/static/rdbm/ \
+    && cp ${JANS_SETUP_DIR}/static/rdbm/opendj_attributes_syntax.json /app/static/rdbm/ \
+    && cp ${JANS_SETUP_DIR}/static/rdbm/sub_tables.json /app/static/rdbm/ \
+    && cp ${JANS_SETUP_DIR}/schema/jans_schema.json /app/schema/ \
+    && cp ${JANS_SETUP_DIR}/schema/custom_schema.json /app/schema/ \
+    && cp ${JANS_SETUP_DIR}/schema/opendj_types.json /app/schema/ \
+    && cp ${JANS_SETUP_DIR}/templates/jans-config-api/config.ldif /app/templates/jans-config-api/ \
+    && cp ${JANS_SETUP_DIR}/templates/jans-config-api/dynamic-conf.json /app/templates/jans-config-api/ \
+    && cp ${JANS_CONFIG_API_DOCS}/jans-config-api-swagger-auto.yaml /app/static/
+
 # =======
 # Cleanup
 # =======
@@ -212,9 +258,13 @@ RUN touch /etc/hosts.back
 COPY jetty/log4j2.xml ${JETTY_BASE}/jans-config-api/resources/
 COPY conf/*.tmpl /app/templates/
 COPY plugins /app/plugins
+COPY templates /app/templates
 COPY scripts /app/scripts
 RUN chmod +x /app/scripts/entrypoint.sh
 
+# unquote apiApprovedIssuer
+RUN sed -i 's/"${apiApprovedIssuer}"/${apiApprovedIssuer}/g' /app/templates/jans-config-api/dynamic-conf.json
+
 # create non-root user
 RUN adduser -s /bin/sh -D -G root -u 1000 jetty
 
@@ -229,7 +279,8 @@ RUN chmod -R g=u ${JETTY_BASE}/jans-config-api/custom \
     && chmod 664 /etc/hosts.back \
     && chmod 664 /usr/java/latest/jre/lib/security/cacerts \
     && chmod 664 /opt/jetty/etc/jetty.xml \
-    && chmod 664 /opt/jetty/etc/webdefault.xml
+    && chmod 664 /opt/jetty/etc/webdefault.xml \
+    && chmod -R g=u /app/templates/jans-config-api
 
 USER 1000
 

docker-jans-config-api/README.md

@@ -70,6 +70,7 @@ The following environment variables are supported by the container:
 - `CN_CONFIG_API_APP_LOGGERS`: Custom logging configuration in JSON-string format with hash type (see [Configure app loggers](#configure-app-loggers) section for details).
 - `CN_CONFIG_API_PLUGINS`: Comma-separated plugin names that should be enabled (available plugins are `admin-ui`, `scim`, `fido2`, and `user-mgt`). Note that unknown plugin name will be ignored.
 - `CN_TOKEN_SERVER_CERT_FILE`: Path to token server certificate (default to `/etc/certs/token_server.crt`).
+- `CN_TOKEN_SERVER_BASE_HOSTNAME`: Hostname of token server (default to empty string).
 - `CN_ADMIN_UI_PLUGIN_LOGGERS`: Custom logging configuration for AdminUI plugin in JSON-string format with hash type (see [Configure plugin loggers](#configure-plugin-loggers) section for details).
 - `CN_PROMETHEUS_PORT`: Port used by Prometheus JMX agent (default to empty string). To enable Prometheus JMX agent, set the value to a number. See [Exposing metrics](#exposing-metrics) for details.
 - `CN_SQL_DB_HOST`: Hostname of the SQL database (default to `localhost`).

docker-jans-config-api/requirements.txt

@@ -1,4 +1,5 @@
 # pinned to py3-grpcio version to avoid failure on native extension build
 grpcio==1.41.0
 libcst<0.4
-git+https://github.com/JanssenProject/jans@9b536ab2b5d398a41733790f2eeb70339f993fb7#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+ruamel.yaml==0.16.10
+git+https://github.com/JanssenProject/jans@42f2834680de9578bce5f39ac5de35de19a6c48d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-config-api/scripts/bootstrap.py

@@ -2,7 +2,13 @@
 import logging.config
 import os
 import re
+import typing as _t
+from functools import cached_property
 from string import Template
+from urllib.parse import urlparse
+from uuid import uuid4
+
+from ldif import LDIFWriter
 
 from jans.pycloudlib import get_manager
 from jans.pycloudlib.persistence import render_couchbase_properties
@@ -14,12 +20,23 @@
 from jans.pycloudlib.persistence import sync_ldap_truststore
 from jans.pycloudlib.persistence import render_sql_properties
 from jans.pycloudlib.persistence import render_spanner_properties
+from jans.pycloudlib.persistence.couchbase import CouchbaseClient
+from jans.pycloudlib.persistence.couchbase import id_from_dn
+from jans.pycloudlib.persistence.ldap import LdapClient
+from jans.pycloudlib.persistence.spanner import SpannerClient
+from jans.pycloudlib.persistence.sql import SqlClient
+from jans.pycloudlib.persistence.sql import doc_id_from_dn
 from jans.pycloudlib.persistence.utils import PersistenceMapper
 from jans.pycloudlib.utils import cert_to_truststore
+from jans.pycloudlib.utils import generate_base64_contents
+from jans.pycloudlib.utils import get_random_chars
+from jans.pycloudlib.utils import encode_text
 
 from settings import LOGGING_CONFIG
 from plugins import AdminUiPlugin
 from plugins import discover_plugins
+from utils import parse_config_api_swagger
+from utils import generate_hex
 
 logging.config.dictConfig(LOGGING_CONFIG)
 logger = logging.getLogger("entrypoint")
@@ -89,6 +106,9 @@ def main():
     modify_webdefault_xml()
     configure_logging()
 
+    persistence_setup = PersistenceSetup(manager)
+    persistence_setup.import_ldif_files()
+
     plugins = discover_plugins()
     logger.info(f"Loaded config-api plugins: {', '.join(plugins)}")
     modify_config_api_xml(plugins)
@@ -287,5 +307,188 @@ def configure_admin_ui_logging():
         f.write(tmpl.safe_substitute(config))
 
 
+class PersistenceSetup:
+    def __init__(self, manager) -> None:
+        self.manager = manager
+
+        client_classes = {
+            "ldap": LdapClient,
+            "couchbase": CouchbaseClient,
+            "spanner": SpannerClient,
+            "sql": SqlClient,
+        }
+
+        # determine persistence type
+        mapper = PersistenceMapper()
+        self.persistence_type = mapper.mapping["default"]
+
+        # determine persistence client
+        client_cls = client_classes.get(self.persistence_type)
+        self.client = client_cls(manager)
+
+    def get_auth_config(self):
+        dn = "ou=jans-auth,ou=configuration,o=jans"
+
+        # sql and spanner
+        if self.persistence_type in ("sql", "spanner"):
+            entry = self.client.get("jansAppConf", doc_id_from_dn(dn))
+            return json.loads(entry["jansConfDyn"])
+
+        # couchbase
+        elif self.persistence_type == "couchbase":
+            key = id_from_dn(dn)
+            bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
+            req = self.client.exec_query(
+                f"SELECT META().id, {bucket}.* FROM {bucket} USE KEYS '{key}'"
+            )
+            attrs = req.json()["results"][0]
+            return attrs["jansConfDyn"]
+
+        # ldap
+        else:
+            entry = self.client.get(dn, attributes=["jansConfDyn"])
+            return json.loads(entry.entry_attributes_as_dict["jansConfDyn"][0])
+
+    def transform_url(self, url):
+        auth_server_url = os.environ.get("CN_AUTH_SERVER_URL", "")
+
+        if not auth_server_url:
+            return url
+
+        parse_result = urlparse(url)
+        if parse_result.path.startswith("/.well-known"):
+            path = f"/jans-auth{parse_result.path}"
+        else:
+            path = parse_result.path
+        url = f"http://{auth_server_url}{path}"
+        return url
+
+    def get_injected_urls(self):
+        auth_config = self.get_auth_config()
+
+        urls = (
+            "issuer",
+            "openIdConfigurationEndpoint",
+            "introspectionEndpoint",
+            "tokenEndpoint",
+            "tokenRevocationEndpoint",
+        )
+
+        return {
+            url: self.transform_url(auth_config[url])
+            for url in urls
+        }
+
+    @cached_property
+    def ctx(self) -> dict[str, _t.Any]:
+        hostname = self.manager.config.get("hostname")
+        approved_issuer = [hostname]
+
+        token_server_hostname = os.environ.get("CN_TOKEN_SERVER_BASE_HOSTNAME")
+        if token_server_hostname and token_server_hostname not in approved_issuer:
+            approved_issuer.append(token_server_hostname)
+
+        ctx = {
+            "hostname": hostname,
+            "apiApprovedIssuer": ",".join([f'"https://{issuer}"' for issuer in approved_issuer]),
+            "apiProtectionType": "oauth2",
+            "endpointInjectionEnabled": "true",
+            "configOauthEnabled": str(os.environ.get("CN_CONFIG_API_OAUTH_ENABLED") or True).lower(),
+        }
+        ctx.update(self.get_injected_urls())
+
+        # Client
+        ctx["jca_client_id"] = self.manager.config.get("jca_client_id")
+        if not ctx["jca_client_id"]:
+            ctx["jca_client_id"] = f"1800.{uuid4()}"
+            self.manager.config.set("jca_client_id", ctx["jca_client_id"])
+
+        ctx["jca_client_pw"] = self.manager.secret.get("jca_client_pw")
+        if not ctx["jca_client_pw"]:
+            ctx["jca_client_pw"] = get_random_chars()
+            self.manager.secret.set("jca_client_pw", ctx["jca_client_pw"])
+
+        ctx["jca_client_encoded_pw"] = self.manager.secret.get("jca_client_encoded_pw")
+        if not ctx["jca_client_encoded_pw"]:
+            ctx["jca_client_encoded_pw"] = encode_text(
+                ctx["jca_client_pw"], self.manager.secret.get("encoded_salt"),
+            ).decode()
+            self.manager.secret.set("jca_client_encoded_pw", ctx["jca_client_encoded_pw"])
+
+        # pre-populate config_api_dynamic_conf_base64
+        with open("/app/templates/jans-config-api/dynamic-conf.json") as f:
+            tmpl = Template(f.read())
+            ctx["config_api_dynamic_conf_base64"] = generate_base64_contents(tmpl.substitute(**ctx))
+
+        # finalize ctx
+        return ctx
+
+    def get_scope_jans_ids(self):
+        if self.persistence_type in ("sql", "spanner"):
+            entries = self.client.search("jansScope", ["jansId"])
+            return [entry["jansId"] for entry in entries]
+
+        if self.persistence_type == "couchbase":
+            bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
+            req = self.client.exec_query(
+                f"SELECT {bucket}.jansId FROM {bucket} WHERE objectClass = 'jansScope'",
+            )
+            results = req.json()["results"]
+            return [item["jansId"] for item in results]
+
+        # likely ldap
+        entries = self.client.search("ou=scopes,o=jans", "(objectClass=jansScope)", ["jansId"])
+        return [entry.entry_attributes_as_dict["jansId"][0] for entry in entries]
+
+    def generate_scopes_ldif(self):
+        # jansId to compare to
+        existing_jans_ids = self.get_scope_jans_ids()
+
+        def generate_config_api_scopes():
+            swagger = parse_config_api_swagger()
+            scopes = swagger["components"]["securitySchemes"]["oauth2"]["flows"]["clientCredentials"]["scopes"]
+
+            generated_scopes = []
+            for jans_id, desc in scopes.items():
+                if jans_id in existing_jans_ids:
+                    continue
+
+                inum = f"1800.{generate_hex()}-{generate_hex()}"
+                attrs = {
+                    "creatorAttrs": [json.dumps({})],
+                    "description": [desc],
+                    "displayName": [f"Config API scope {jans_id}"],
+                    "inum": [inum],
+                    "jansAttrs": [json.dumps({"spontaneousClientScopes": None, "showInConfigurationEndpoint": True})],
+                    "jansId": [jans_id],
+                    "jansScopeTyp": ["oauth"],
+                    "objectClass": ["top", "jansScope"],
+                    "jansDefScope": ["false"],
+                }
+                generated_scopes.append(attrs)
+            return generated_scopes
+
+        # prepare required scopes (if any)
+        scopes = []
+
+        config_api_scopes = generate_config_api_scopes()
+        scopes += config_api_scopes
+
+        with open("/app/templates/jans-config-api/scopes.ldif", "wb") as fd:
+            writer = LDIFWriter(fd, cols=1000)
+            for scope in scopes:
+                writer.unparse(f"inum={scope['inum'][0]},ou=scopes,o=jans", scope)
+
+    def import_ldif_files(self) -> None:
+        self.generate_scopes_ldif()
+
+        files = ["config.ldif", "scopes.ldif", "clients.ldif"]
+        ldif_files = [f"/app/templates/jans-config-api/{file_}" for file_ in files]
+
+        for file_ in ldif_files:
+            logger.info(f"Importing {file_}")
+            self.client.create_from_ldif(file_, self.ctx)
+
+
 if __name__ == "__main__":
     main()

docker-jans-config-api/scripts/entrypoint.sh

@@ -24,6 +24,7 @@ get_prometheus_opt() {
 
 python3 /app/scripts/wait.py
 python3 /app/scripts/bootstrap.py
+python3 /app/scripts/upgrade.py
 
 # run config-api
 cd /opt/jans/jetty/jans-config-api
@@ -35,6 +36,7 @@ exec java \
     -Djans.base=/etc/jans \
     -Dserver.base=/opt/jans/jetty/jans-config-api \
     -Dlog.base=/opt/jans/jetty/jans-config-api \
+    -Dpython.home=/opt/jython \
     -Djava.io.tmpdir=/tmp \
     -Dlog4j2.configurationFile=$(get_logging_files) \
     $(get_prometheus_opt) \