From b45c84571360a8e6f1de9c5d3688e608c9e1284d Mon Sep 17 00:00:00 2001 From: Yuriy Movchan Date: Fri, 30 Nov 2018 14:13:51 +0300 Subject: [PATCH] Store metric records in separate backed o=metric oxAuth #934 --- .../oxtrust/ldap/service/AppInitializer.java | 177 +++++++++++++----- .../gluu/oxtrust/service/CleanerTimer.java | 2 +- .../gluu/oxtrust/service/MetricService.java | 12 +- 3 files changed, 139 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/org/gluu/oxtrust/ldap/service/AppInitializer.java b/server/src/main/java/org/gluu/oxtrust/ldap/service/AppInitializer.java index 3edd65878..ad5f79905 100644 --- a/server/src/main/java/org/gluu/oxtrust/ldap/service/AppInitializer.java +++ b/server/src/main/java/org/gluu/oxtrust/ldap/service/AppInitializer.java @@ -6,8 +6,8 @@ package org.gluu.oxtrust.ldap.service; -import com.unboundid.ldap.sdk.ResultCode; import java.io.IOException; +import java.lang.annotation.Annotation; import java.net.URISyntaxException; import java.util.Arrays; import java.util.List; @@ -27,9 +27,6 @@ import javax.inject.Named; import javax.servlet.ServletContext; -import com.unboundid.ldap.sdk.ResultCode; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.LoggerContext; import org.gluu.oxtrust.config.ConfigurationFactory; import org.gluu.oxtrust.ldap.cache.service.CacheRefreshTimer; import org.gluu.oxtrust.service.CleanerTimer; @@ -43,7 +40,6 @@ import org.gluu.site.ldap.persistence.LdapEntryManager; import org.slf4j.Logger; import org.slf4j.bridge.SLF4JBridgeHandler; -import org.xdi.config.oxtrust.AppConfiguration; import org.xdi.exception.ConfigurationException; import org.xdi.exception.OxIntializationException; import org.xdi.model.custom.script.CustomScriptType; @@ -58,13 +54,13 @@ import org.xdi.oxauth.model.util.SecurityProviderUtility; import org.xdi.service.JsonService; import org.xdi.service.PythonService; -import org.xdi.service.cdi.event.ConfigurationUpdate; import org.xdi.service.cdi.event.LdapConfigurationReload; -import org.xdi.service.cdi.event.LoggerUpdateEvent; import org.xdi.service.cdi.util.CdiUtil; import org.xdi.service.custom.lib.CustomLibrariesLoader; import org.xdi.service.custom.script.CustomScriptManager; +import org.xdi.service.ldap.LdapConnectionProviders; import org.xdi.service.ldap.LdapConnectionService; +import org.xdi.service.metric.inject.ReportMetric; import org.xdi.service.timer.QuartzSchedulerManager; import org.xdi.service.timer.event.TimerEvent; import org.xdi.util.StringHelper; @@ -72,6 +68,8 @@ import org.xdi.util.security.StringEncrypter; import org.xdi.util.security.StringEncrypter.EncryptionException; +import com.unboundid.ldap.sdk.ResultCode; + /** * Perform startup time initialization * @@ -81,7 +79,10 @@ @Named public class AppInitializer { + public static final String LDAP_METRIC_CONFIG_GROUP_NAME = "metric"; + public static final String LDAP_ENTRY_MANAGER_NAME = "ldapEntryManager"; + public static final String LDAP_METRIC_ENTRY_MANAGER_NAME = "ldapMetricEntryManager"; public static final String LDAP_CENTRAL_ENTRY_MANAGER_NAME = "centralLdapEntryManager"; @Inject @@ -99,6 +100,9 @@ public class AppInitializer { @Inject @Named(LDAP_ENTRY_MANAGER_NAME) private Instance ldapEntryManagerInstance; + @Inject @Named(LDAP_METRIC_ENTRY_MANAGER_NAME) @ReportMetric + private Instance ldapMetricEntryManagerInstance; + @Inject @Named(LDAP_CENTRAL_ENTRY_MANAGER_NAME) @CentralLdap private Instance ldapCentralEntryManagerInstance; @@ -176,8 +180,10 @@ public class AppInitializer { private FileConfiguration ldapConfig; private FileConfiguration ldapCentralConfig; - private LdapConnectionService connectionProvider; - private LdapConnectionService centralConnectionProvider; + + private LdapConnectionProviders connectionProviders; + private LdapConnectionProviders metricConnectionProviders; + private LdapConnectionProviders centralConnectionProviders; private AtomicBoolean isActive; private long lastFinishedTime; @@ -203,14 +209,15 @@ public void applicationInitialized(@Observes @Initialized(ApplicationScoped.clas customLibrariesLoader.init(); // Initialize local LDAP connection provider - createConnectionProvider(); - + this.connectionProviders = createConnectionProvider((String) null, false); + this.metricConnectionProviders = createConnectionProvider(LDAP_METRIC_CONFIG_GROUP_NAME, false); + configurationFactory.create(); LdapEntryManager localLdapEntryManager = ldapEntryManagerInstance.get(); // Initialize central LDAP connection provider - createCentralConnectionProvider(); + this.centralConnectionProviders = createCentralConnectionProvider(); initializeLdifArchiver(localLdapEntryManager); @@ -279,12 +286,14 @@ public StringEncrypter getStringEncrypter() throws OxIntializationException { public void destroy(@Observes @BeforeDestroyed(ApplicationScoped.class) ServletContext init) { log.info("Closing LDAP connection at server shutdown..."); LdapEntryManager ldapEntryManager = ldapEntryManagerInstance.get(); - closeLdapEntryManager(ldapEntryManager); + closeLdapEntryManager(ldapEntryManager, LDAP_ENTRY_MANAGER_NAME); + LdapEntryManager ldapMetricEntryManager = ldapMetricEntryManagerInstance.get(); + closeLdapEntryManager(ldapMetricEntryManager, LDAP_METRIC_ENTRY_MANAGER_NAME); LdapEntryManager ldapCentralEntryManager = ldapCentralEntryManagerInstance.get(); if (ldapCentralEntryManager != null) { - closeLdapEntryManager(ldapCentralEntryManager); + closeLdapEntryManager(ldapCentralEntryManager, LDAP_CENTRAL_ENTRY_MANAGER_NAME); } } @@ -293,25 +302,28 @@ private void showBuildInfo() { getGluuRevisionVersion(), getGluuRevisionDate(), getGluuBuildNumber()); } - private void createConnectionProvider() { - this.ldapConfig = configurationFactory.getLdapConfiguration(); - - Properties connectionProperties = (Properties) this.ldapConfig.getProperties(); - this.connectionProvider = createConnectionProvider(connectionProperties); - if (!ResultCode.SUCCESS.equals(this.connectionProvider.getCreationResultCode())) { - throw new ConfigurationException("Failed to create LDAP connection pool!"); - } - log.debug("Created connectionProvider: {}", connectionProvider); - } - - private void createCentralConnectionProvider() { + private LdapConnectionProviders createCentralConnectionProvider() { if ((configurationFactory.getLdapCentralConfiguration() != null) && configurationFactory.getAppConfiguration().isUpdateApplianceStatus()) { this.ldapCentralConfig = configurationFactory.getLdapCentralConfiguration(); Properties connectionProperties = (Properties) this.ldapConfig.getProperties(); - this.centralConnectionProvider = createConnectionProvider(connectionProperties); + LdapConnectionService centralConnectionProvider = createConnectionProvider(connectionProperties); log.debug("Created centralConnectionProvider: {}", centralConnectionProvider); + + return new LdapConnectionProviders(centralConnectionProvider, null); } + + return null; + } + + + private Properties prepareBindConnectionProperties(Properties connectionProperties) { + // TODO: Use own properties with prefix specified in variable 'bindConfigurationComponentName' + Properties bindProperties = (Properties) connectionProperties.clone(); + bindProperties.remove("bindDN"); + bindProperties.remove("bindPassword"); + + return bindProperties; } private LdapConnectionService createConnectionProvider(Properties connectionProperties) { @@ -321,40 +333,115 @@ private LdapConnectionService createConnectionProvider(Properties connectionProp return connectionProvider; } - @Produces @ApplicationScoped @Named(LDAP_ENTRY_MANAGER_NAME) - public LdapEntryManager getLdapEntryManager() { - LdapEntryManager ldapEntryManager = new LdapEntryManager(new OperationsFacade(this.connectionProvider)); - log.info("Created {}: {}", new Object[] { LDAP_ENTRY_MANAGER_NAME, ldapEntryManager.getLdapOperationService() }); + private LdapConnectionService createBindConnectionProvider(Properties bindConnectionProperties, Properties connectionProperties) { + LdapConnectionService bindConnectionProvider = createConnectionProvider(bindConnectionProperties); + if (ResultCode.INAPPROPRIATE_AUTHENTICATION.equals(bindConnectionProvider.getCreationResultCode())) { + log.warn("It's not possible to create authentication LDAP connection pool using anonymous bind. Attempting to create it using binDN/bindPassword"); + bindConnectionProvider = createConnectionProvider(connectionProperties); + } + + return bindConnectionProvider; + } + @Produces @ApplicationScoped @Named(LDAP_ENTRY_MANAGER_NAME) + public LdapEntryManager createLdapEntryManager() { + LdapEntryManager ldapEntryManager = new LdapEntryManager(new OperationsFacade(this.connectionProviders.getConnectionProvider())); + log.info("Created {}:{} with provider {}", LDAP_ENTRY_MANAGER_NAME, ldapEntryManager, ldapEntryManager.getLdapOperationService().getConnectionProvider() ); + return ldapEntryManager; + } + + @Produces @Named(LDAP_METRIC_ENTRY_MANAGER_NAME) @ReportMetric @ApplicationScoped + public LdapEntryManager createLdapMetricEntryManager() { + LdapEntryManager ldapMetricEntryManager = new LdapEntryManager(new OperationsFacade(this.metricConnectionProviders.getConnectionProvider())); + log.info("Created {}:{} with provider {}", LDAP_METRIC_ENTRY_MANAGER_NAME, ldapMetricEntryManager, ldapMetricEntryManager.getLdapOperationService().getConnectionProvider()); + + return ldapMetricEntryManager; } @Produces @ApplicationScoped @Named(LDAP_CENTRAL_ENTRY_MANAGER_NAME) @CentralLdap public LdapEntryManager createCentralLdapEntryManager() { - if (this.centralConnectionProvider == null) { + if (this.centralConnectionProviders == null) { return new LdapEntryManager(); } - LdapEntryManager centralLdapEntryManager = new LdapEntryManager(new OperationsFacade(this.centralConnectionProvider)); - log.info("Created {}: {}", new Object[] { LDAP_CENTRAL_ENTRY_MANAGER_NAME, centralLdapEntryManager.getLdapOperationService() }); + LdapEntryManager centralLdapEntryManager = new LdapEntryManager(new OperationsFacade(this.centralConnectionProviders.getConnectionProvider())); + log.info("Created {}:{} with provider {}", LDAP_CENTRAL_ENTRY_MANAGER_NAME, centralLdapEntryManager, centralLdapEntryManager.getLdapOperationService().getConnectionProvider()); return centralLdapEntryManager; } + public void recreateLdapEntryManager(@Observes @LdapConfigurationReload String event) { + this.connectionProviders = recreateLdapEntryManagerImpl(LDAP_ENTRY_MANAGER_NAME, null, false); + forceCreateNewEntryManager(ldapEntryManagerInstance, LDAP_ENTRY_MANAGER_NAME); + + this.metricConnectionProviders = recreateLdapEntryManagerImpl(LDAP_METRIC_ENTRY_MANAGER_NAME, LDAP_METRIC_CONFIG_GROUP_NAME, false, ReportMetric.Literal.INSTANCE); + forceCreateNewEntryManager(ldapMetricEntryManagerInstance, LDAP_METRIC_ENTRY_MANAGER_NAME); + } + + protected LdapConnectionProviders recreateLdapEntryManagerImpl(String entryManagerName, String configId, boolean createBind, Annotation... qualifiers) { // Get existing application scoped instance - LdapEntryManager oldLdapEntryManager = CdiUtil.getContextBean(beanManager, LdapEntryManager.class, LDAP_ENTRY_MANAGER_NAME); + LdapEntryManager oldLdapEntryManager = CdiUtil.getContextBean(beanManager, LdapEntryManager.class, entryManagerName, qualifiers); // Recreate components - createConnectionProvider(); + LdapConnectionProviders createConnectionProviders = createConnectionProvider(configId, createBind); // Close existing connections - closeLdapEntryManager(oldLdapEntryManager); + closeLdapEntryManager(oldLdapEntryManager, entryManagerName); + + return createConnectionProviders; + } + protected void forceCreateNewEntryManager(Instance instance, String entryManagerName) { // Force to create new bean - LdapEntryManager ldapEntryManager = ldapEntryManagerInstance.get(); - ldapEntryManagerInstance.destroy(ldapEntryManager); - log.info("Recreated instance {}: {}", LDAP_ENTRY_MANAGER_NAME, ldapEntryManager); + T ldapEntryManager = instance.get(); + instance.destroy(ldapEntryManager); + log.info("Recreated instance {}: {}", entryManagerName, ldapEntryManager); + } + + private LdapConnectionProviders createConnectionProvider(String configId, boolean createBind) { + Properties connectionProperties = getLdapConfigProperties(configId); + String logConfigId = StringHelper.isEmpty(configId) ? "" : configId + "-"; + + LdapConnectionService connectionProvider = createConnectionProvider(connectionProperties); + if (!ResultCode.SUCCESS.equals(connectionProvider.getCreationResultCode())) { + throw new ConfigurationException("Failed to create LDAP connection pool!"); + } + log.debug("Created {}connectionProvider: {}", logConfigId, connectionProvider); + + LdapConnectionService bindConnectionProvider = null; + if (createBind) { + Properties bindConnectionProperties = prepareBindConnectionProperties(connectionProperties); + bindConnectionProvider = createBindConnectionProvider(bindConnectionProperties, connectionProperties); + if (!ResultCode.SUCCESS.equals(bindConnectionProvider.getCreationResultCode())) { + throw new ConfigurationException("Failed to create LDAP connection pool!"); + } + log.debug("Created {}bindConnectionProvider: {}", logConfigId, bindConnectionProvider); + } + + return new LdapConnectionProviders(connectionProvider, bindConnectionProvider); + } + + protected Properties getLdapConfigProperties(String configId) { + Properties connectionProperties = (Properties) configurationFactory.getLdapConfiguration().getProperties(); + if (StringHelper.isNotEmpty(configId)) { + // Replace properties names 'configId.xyz' to 'configId.xyz' in order to override default values + connectionProperties = (Properties) connectionProperties.clone(); + + String baseGroup = configId + "."; + for (Object key : connectionProperties.keySet()) { + String propertyName = (String) key; + if (propertyName.startsWith(baseGroup)) { + propertyName = propertyName.substring(baseGroup.length()); + + Object value = connectionProperties.get(key); + connectionProperties.put(propertyName, value); + } + } + } + + return connectionProperties; } public void recreateCentralLdapEntryManager(@Observes @LdapCentralConfigurationReload String event) { @@ -362,10 +449,10 @@ public void recreateCentralLdapEntryManager(@Observes @LdapCentralConfigurationR LdapEntryManager oldCentralLdapEntryManager = CdiUtil.getContextBean(beanManager, LdapEntryManager.class, LDAP_CENTRAL_ENTRY_MANAGER_NAME); // Recreate components - createCentralConnectionProvider(); + this.centralConnectionProviders = createCentralConnectionProvider(); // Close existing connections - closeLdapEntryManager(oldCentralLdapEntryManager); + closeLdapEntryManager(oldCentralLdapEntryManager, LDAP_CENTRAL_ENTRY_MANAGER_NAME); // Force to create new bean LdapEntryManager ldapCentralEntryManager = ldapCentralEntryManagerInstance.get(); @@ -373,12 +460,12 @@ public void recreateCentralLdapEntryManager(@Observes @LdapCentralConfigurationR log.info("Recreated instance {}: {}", LDAP_CENTRAL_ENTRY_MANAGER_NAME, ldapCentralEntryManager); } - private void closeLdapEntryManager(LdapEntryManager oldLdapEntryManager) { + private void closeLdapEntryManager(LdapEntryManager oldLdapEntryManager, String entryManagerName) { // Close existing connections if ((oldLdapEntryManager != null) && (oldLdapEntryManager.getLdapOperationService() != null)) { - log.debug("Attempting to destroy {}: {}", LDAP_ENTRY_MANAGER_NAME, oldLdapEntryManager); + log.debug("Attempting to destroy {}:{} with provider {}", entryManagerName, oldLdapEntryManager, oldLdapEntryManager.getLdapOperationService().getConnectionProvider()); oldLdapEntryManager.destroy(); - log.debug("Destroyed {}: {}", LDAP_ENTRY_MANAGER_NAME, oldLdapEntryManager); + log.debug("Destroyed {}:{} with provider {}", entryManagerName, oldLdapEntryManager, oldLdapEntryManager.getLdapOperationService().getConnectionProvider()); } } diff --git a/server/src/main/java/org/gluu/oxtrust/service/CleanerTimer.java b/server/src/main/java/org/gluu/oxtrust/service/CleanerTimer.java index b0d429eed..13ba7c9e9 100644 --- a/server/src/main/java/org/gluu/oxtrust/service/CleanerTimer.java +++ b/server/src/main/java/org/gluu/oxtrust/service/CleanerTimer.java @@ -127,7 +127,7 @@ private void processMetricEntries() { cleanUpLogger.addNewLogLine(String.format( "#Ready to remove expired entries with parameters batch size: %s, expiration date: %s and appliance inum %s", BATCH_SIZE, expirationDate, metricService.applianceInum())); - metricService.removeExpiredMetricEntries(BATCH_SIZE, expirationDate, ApplicationType.OX_AUTH, + metricService.removeExpiredMetricEntries(BATCH_SIZE, expirationDate, ApplicationType.OX_TRUST, metricService.applianceInum()); log.debug("End metric entries clean up"); cleanUpLogger.addNewLogLine("#Processing Metric entries done at:" + new Date()); diff --git a/server/src/main/java/org/gluu/oxtrust/service/MetricService.java b/server/src/main/java/org/gluu/oxtrust/service/MetricService.java index 74b5c8b93..7c042e6ad 100644 --- a/server/src/main/java/org/gluu/oxtrust/service/MetricService.java +++ b/server/src/main/java/org/gluu/oxtrust/service/MetricService.java @@ -19,6 +19,7 @@ import javax.inject.Inject; import javax.inject.Named; +import org.gluu.oxtrust.ldap.service.AppInitializer; import org.gluu.oxtrust.ldap.service.ApplianceService; import org.gluu.oxtrust.ldap.service.OrganizationService; import org.gluu.oxtrust.model.AuthenticationChartDto; @@ -31,6 +32,7 @@ import org.xdi.model.metric.counter.CounterMetricEntry; import org.xdi.model.metric.ldap.MetricEntry; import org.xdi.service.CacheService; +import org.xdi.service.metric.inject.ReportMetric; /** * Store and retrieve metric @@ -67,8 +69,8 @@ public class MetricService extends org.xdi.service.metric.MetricService { @Inject private Logger logger; - @Inject - private LdapEntryManager ldapEntryManager; + @Inject @Named(AppInitializer.LDAP_METRIC_ENTRY_MANAGER_NAME) @ReportMetric + private LdapEntryManager ldapMetricEntryManager; public void initTimer() { initTimer(this.appConfiguration.getMetricReporterInterval()); @@ -76,9 +78,7 @@ public void initTimer() { @Override public String baseDn() { - String orgDn = organizationService.getDnForOrganization(); - String baseDn = String.format("ou=metric,%s", orgDn); - return baseDn; + return "ou=statistic,o=metric"; } @Override @@ -259,7 +259,7 @@ private void dump(List metrics) { } public LdapEntryManager getEntryManager() { - return ldapEntryManager; + return ldapMetricEntryManager; } }