feat: update config-api image (#874)

* fix: update dependencies

Overview:

- update jans-config-api-server
- update URLs of downloadable plugins
- add facter script

* feat: update logging configuration

* feat: add logging configuration for admin-ui plugin

docker-jans-config-api/Dockerfile

@@ -32,7 +32,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
 # ==========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2022-01-25 07:53'
+ENV CN_BUILD_DATE='2022-02-22 05:22'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API
@@ -46,18 +46,26 @@ RUN wget -q ${CN_SOURCE_URL} -O /tmp/jans-config-api.war \
 # Ports exposed by config-api
 EXPOSE 8074
 
+# ======
+# Facter
+# ======
+
+ARG PYFACTER_VERSION=9d8478ee47dc5498a766e010e8d3a3451b46e541
+RUN wget -q https://github.com/GluuFederation/gluu-snap/raw/${PYFACTER_VERSION}/facter/facter -O /usr/bin/facter \
+    && chmod +x /usr/bin/facter
+
 # =======
 # Plugins
 # =======
 
 RUN mkdir -p /usr/share/java
 
-ENV SCIM_PLUGIN_BUILD_DATE='2022-01-25 07:54'
-ENV SCIM_PLUGIN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/scim-plugin/${CN_VERSION}/scim-plugin-${CN_VERSION}-distribution.jar
+ENV SCIM_PLUGIN_BUILD_DATE='2022-02-21 16:26'
+ENV SCIM_PLUGIN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api/plugins/scim-plugin/${CN_VERSION}/scim-plugin-${CN_VERSION}-distribution.jar
 RUN wget -q ${SCIM_PLUGIN_SOURCE_URL} -O /usr/share/java/scim-plugin.jar
 
-ENV ADMIN_UI_PLUGIN_BUILD_DATE='2022-01-25 07:53'
-ENV ADMIN_UI_SOURCE_URL=https://maven.jans.io/maven/io/jans/admin-ui-plugin/${CN_VERSION}/admin-ui-plugin-${CN_VERSION}-distribution.jar
+ENV ADMIN_UI_PLUGIN_BUILD_DATE='2022-02-21 16:26'
+ENV ADMIN_UI_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api/plugins/admin-ui-plugin/${CN_VERSION}/admin-ui-plugin-${CN_VERSION}-distribution.jar
 RUN wget -q ${ADMIN_UI_SOURCE_URL} -O /usr/share/java/admin-ui-plugin.jar
 
 # ======

docker-jans-config-api/README.md

@@ -80,6 +80,7 @@ The following environment variables are supported by the container:
 - `CN_LICENSE_ADMIN_UI_PRODUCT_CODE`: Path to admin-ui license product code (default to `/etc/jans/conf/admin_ui_product_code`).
 - `CN_LICENSE_ADMIN_UI_SHARED_KEY`: Path to admin-ui license shared key (default to `/etc/jans/conf/admin_ui_shared_key`).
 - `CN_LICENSE_ADMIN_UI_MANAGEMENT_KEY`: Path to admin-ui license management key (default to `/etc/jans/conf/admin_ui_management_key`).
+- `CN_ADMIN_UI_PLUGIN_LOGGERS`: Custom logging configuration for AdminUI plugin in JSON-string format with hash type (see [Configure plugin loggers](#configure-plugin-loggers) section for details).
 
 ### Configure app loggers
 
@@ -104,6 +105,43 @@ The following key-value pairs are the defaults:
 ```json
 {
     "config_api_log_target": "STDOUT",
-    "config_api_log_level": "INFO"
+    "config_api_log_level": "INFO",
+    "persistence_log_target": "FILE",
+    "persistence_log_level": "INFO",
+    "persistence_duration_log_target": "FILE",
+    "persistence_duration_log_level": "INFO",
+    "ldap_stats_log_target": "FILE",
+    "ldap_stats_log_level": "INFO",
+    "script_log_target": "FILE",
+    "script_log_level": "INFO"
+}
+```
+
+### Configure plugin loggers
+
+Plugin loggers can be configured to define where the logs will be redirected and what is the level the logs should be displayed.
+
+Supported redirect target:
+
+- `STDOUT`
+- `FILE`
+
+Supported level:
+
+- `FATAL`
+- `ERROR`
+- `WARN`
+- `INFO`
+- `DEBUG`
+- `TRACE`
+
+The following key-value pairs are the defaults:
+
+```json
+{
+    "admin_ui_log_target": "FILE",
+    "admin_ui_log_level": "INFO",
+    "admin_ui_audit_log_target": "FILE",
+    "admin_ui_audit_log_level": "INFO"
 }
 ```

docker-jans-config-api/jetty/log4j2.xml

@@ -1,35 +1,111 @@
 <?xml version="1.0" encoding="UTF-8"?>
-
-<Configuration packages="io.jans.log">
+<Configuration status="ERROR">
     <Appenders>
-        <Console name="STDOUT" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p %C{4} %F:%L- %m%n" />
         </Console>
 
         <RollingFile name="FILE" fileName="${sys:log.base}/logs/configapi.log" filePattern="${sys:log.base}/logs/configapi-%d{yyyy-MM-dd}-%i.log">
-            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
+            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p [%t] %C{4} %F:%L- %m%n" />
+
+            <Policies>
+                <SizeBasedTriggeringPolicy size="5 MB" />
+            </Policies>
+            <DefaultRolloverStrategy max="30" />
+        </RollingFile>
+
+        <RollingFile name="JANS_CONFIGAPI_PERSISTENCE_FILE" fileName="${sys:log.base}/logs/configapi_persistence.log" filePattern="${sys:log.base}/logs/configapi_persistence-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
 
             <Policies>
                 <TimeBasedTriggeringPolicy interval="1" modulate="true" />
-                <SizeBasedTriggeringPolicy size="250 MB" />
+                <SizeBasedTriggeringPolicy size="5 MB" />
             </Policies>
+            <DefaultRolloverStrategy max="30" />
+        </RollingFile>
+
+        <RollingFile name="JANS_CONFIGAPI_PERSISTENCE_DURATION_FILE" fileName="${sys:log.base}/logs/configapi_persistence_duration.log" filePattern="${sys:log.base}/logs/configapi_persistence_duration-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
 
-            <DefaultRolloverStrategy max="15" />
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="5 MB" />
+            </Policies>
+            <DefaultRolloverStrategy max="30" />
         </RollingFile>
 
+        <RollingFile name="JANS_CONFIGAPI_PERSISTENCE_LDAP_STATISTICS_FILE" fileName="${sys:log.base}/logs/configapi_persistence_ldap_statistics.log" filePattern="${sys:log.base}/logs/configapi_persistence_ldap_statistics-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
 
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="5 MB" />
+            </Policies>
+            <DefaultRolloverStrategy max="30" />
+        </RollingFile>
+
+        <RollingFile name="JANS_CONFIGAPI_SCRIPT_LOG_FILE" fileName="${sys:log.base}/logs/configapi_script.log" filePattern="${sys:log.base}/logs/configapi_script-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
+
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="5 MB" />
+            </Policies>
+            <DefaultRolloverStrategy max="30" />
+        </RollingFile>
     </Appenders>
 
     <Loggers>
-		<!-- ############### Jans ################# -->
-        <Logger name="io.jans" level="INFO" />
-
         <Logger name="io.jans.configapi" level="$config_api_log_level" additivity="false">
             <AppenderRef ref="$config_api_log_target" />
         </Logger>
 
-        <Root level="INFO">
-			<AppenderRef ref="STDOUT" />
+        <Logger name="io.jans.orm" level="$persistence_log_level" additivity="false">
+            <AppenderRef ref="$persistence_log_target" />
+        </Logger>
+
+        <Logger name="com.unboundid.ldap.sdk.LDAPConnection" level="$persistence_log_level" additivity="false">
+            <AppenderRef ref="$persistence_log_target" />
+        </Logger>
+
+        <logger name="com.couchbase.client" level="$persistence_log_level" additivity="false">
+            <AppenderRef ref="$persistence_log_target" />
+        </logger>
+
+        <Logger name="io.jans.orm.ldap.operation.watch" level="$persistence_duration_log_level" additivity="false">
+            <AppenderRef ref="$persistence_duration_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.orm.couchbase.operation.watch" level="$persistence_duration_log_level" additivity="false">
+            <AppenderRef ref="$persistence_duration_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.orm.watch" level="$persistence_duration_log_level" additivity="false">
+            <AppenderRef ref="$persistence_duration_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.as.server.service.status.ldap" level="$ldap_stats_log_level" additivity="false">
+            <AppenderRef ref="$ldap_stats_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.service.PythonService" level="$script_log_level" additivity="false">
+            <AppenderRef ref="$script_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.service.custom.script" level="$script_log_level" additivity="false">
+            <AppenderRef ref="$script_log_target" />
+        </Logger>
+
+        <Logger name="io.jans.as.server.service.custom" level="$script_log_level" additivity="false">
+            <AppenderRef ref="$script_log_target" />
+        </Logger>
+
+        <logger name="io.jans.service.external" level="$script_log_level" additivity="false">
+            <AppenderRef ref="$script_log_target" />
+        </logger>
+
+        <Root level="ERROR">
+            <AppenderRef ref="Console" />
         </Root>
     </Loggers>
 

docker-jans-config-api/plugins/admin-ui/log4j2-adminui.xml.tmpl

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<Configuration packages="io.jans.log">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p %C{4} %F:%L- %m%n" />
+        </Console>
+
+        <RollingFile name="ADMINUI-AUDIT" fileName="${sys:log.base}/logs/adminuiAudit.log" filePattern="${sys:log.base}/logs/adminuiAudit-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
+
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="250 MB" />
+            </Policies>
+
+            <DefaultRolloverStrategy max="15" />
+        </RollingFile>
+        <RollingFile name="ADMINUI-LOG" fileName="${sys:log.base}/logs/adminui.log" filePattern="${sys:log.base}/logs/adminui-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d{dd-MM HH:mm:ss.SSS} %-5p [%t] [%C{6}] (%F:%L) - %m%n" />
+
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="250 MB" />
+            </Policies>
+
+            <DefaultRolloverStrategy max="15" />
+        </RollingFile>
+
+    </Appenders>
+
+    <Loggers>
+        <Logger name="io.jans.ca.plugin.adminui.rest.logging" level="$admin_ui_audit_log_level" additivity="false">
+            <!-- <AppenderRef ref="ADMINUI-AUDIT" /> -->
+            <AppenderRef ref="$admin_ui_audit_log_target" />
+        </Logger>
+        <Logger name="io.jans.ca.plugin.adminui" level="$admin_ui_log_level" additivity="false">
+            <!-- <AppenderRef ref="ADMINUI-LOG" /> -->
+            <AppenderRef ref="$admin_ui_log_target" />
+        </Logger>
+    </Loggers>
+
+</Configuration>

docker-jans-config-api/scripts/bootstrap.py

@@ -90,6 +90,7 @@ def main():
     if "admin-ui" in plugins:
         admin_ui_plugin = AdminUiPlugin(manager)
         admin_ui_plugin.setup()
+        configure_admin_ui_logging()
 
 
 def modify_jetty_xml():
@@ -140,6 +141,14 @@ def configure_logging():
     config = {
         "config_api_log_target": "STDOUT",
         "config_api_log_level": "INFO",
+        "persistence_log_target": "FILE",
+        "persistence_log_level": "INFO",
+        "persistence_duration_log_target": "FILE",
+        "persistence_duration_log_level": "INFO",
+        "ldap_stats_log_target": "FILE",
+        "ldap_stats_log_level": "INFO",
+        "script_log_target": "FILE",
+        "script_log_level": "INFO",
     }
 
     # pre-populate custom config; format is JSON string of ``dict``
@@ -178,10 +187,20 @@ def configure_logging():
     # mapping between the ``log_target`` value and their appenders
     file_aliases = {
         "config_api_log_target": "FILE",
+        "persistence_log_target": "JANS_CONFIGAPI_PERSISTENCE_FILE",
+        "persistence_duration_log_target": "JANS_CONFIGAPI_PERSISTENCE_DURATION_FILE",
+        "ldap_stats_log_target": "JANS_CONFIGAPI_PERSISTENCE_LDAP_STATISTICS_FILE",
+        "script_log_target": "JANS_CONFIGAPI_SCRIPT_LOG_FILE",
     }
-    for key, value in file_aliases.items():
-        if config[key] == "FILE":
-            config[key] = value
+
+    for key, value in config.items():
+        if not key.endswith("_target"):
+            continue
+
+        if value == "STDOUT":
+            config[key] = "Console"
+        else:
+            config[key] = file_aliases[key]
 
     logfile = "/opt/jans/jetty/jans-config-api/resources/log4j2.xml"
     with open(logfile) as f:
@@ -206,5 +225,70 @@ def modify_config_api_xml(plugins=None):
         f.write(txt % ctx)
 
 
+def configure_admin_ui_logging():
+    # default config
+    config = {
+        "admin_ui_log_target": "FILE",
+        "admin_ui_log_level": "INFO",
+        "admin_ui_audit_log_target": "FILE",
+        "admin_ui_audit_log_level": "INFO",
+    }
+
+    # pre-populate custom config; format is JSON string of ``dict``
+    try:
+        custom_config = json.loads(os.environ.get("CN_ADMIN_UI_PLUGIN_LOGGERS", "{}"))
+    except json.decoder.JSONDecodeError as exc:
+        logger.warning(f"Unable to load logging configuration from environment variable; reason={exc}; fallback to defaults")
+        custom_config = {}
+
+    # ensure custom config is ``dict`` type
+    if not isinstance(custom_config, dict):
+        logger.warning("Invalid data type for CN_CONFIG_API_APP_LOGGERS; fallback to defaults")
+        custom_config = {}
+
+    # list of supported levels; OFF is not supported
+    log_levels = ("FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE",)
+
+    # list of supported outputs
+    log_targets = ("STDOUT", "FILE",)
+
+    for k, v in custom_config.items():
+        if k not in config:
+            continue
+
+        if k.endswith("_log_level") and v not in log_levels:
+            logger.warning(f"Invalid {v} log level for {k}; fallback to defaults")
+            v = config[k]
+
+        if k.endswith("_log_target") and v not in log_targets:
+            logger.warning(f"Invalid {v} log output for {k}; fallback to defaults")
+            v = config[k]
+
+        # update the config
+        config[k] = v
+
+    # mapping between the ``log_target`` value and their appenders
+    file_aliases = {
+        "admin_ui_log_target": "ADMINUI-LOG",
+        "admin_ui_audit_log_target": "ADMINUI-AUDIT",
+    }
+
+    for key, value in config.items():
+        if not key.endswith("_target"):
+            continue
+
+        if value == "STDOUT":
+            config[key] = "Console"
+        else:
+            config[key] = file_aliases[key]
+
+    with open("/app/plugins/admin-ui/log4j2-adminui.xml.tmpl") as f:
+        txt = f.read()
+
+    tmpl = Template(txt)
+    with open("/opt/jans/jetty/jans-config-api/custom/config/log4j2-adminui.xml", "w") as f:
+        f.write(tmpl.safe_substitute(config))
+
+
 if __name__ == "__main__":
     main()