From 1859ed8660219650a54fbcf5504809302c7c6ac9 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Wed, 30 Mar 2016 15:56:22 -0400 Subject: [PATCH] NIFI-1698 Improving customValidate in AbstractHadoopProcessor and HBaseClient service to not reload Configuration unless it changed --- .../org/apache/nifi/hadoop/SecurityUtil.java | 2 +- .../hadoop/AbstractHadoopProcessor.java | 35 +++++++++++++++++-- .../nifi/hbase/HBase_1_1_2_ClientService.java | 35 ++++++++++++++++++- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java b/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java index 74197efa80ca..8d8c6e912019 100644 --- a/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java +++ b/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java @@ -79,7 +79,7 @@ public static synchronized UserGroupInformation loginSimple(final Configuration * @return true if kerberos is enabled on the given configuration, false otherwise * */ - public static synchronized boolean isSecurityEnabled(final Configuration config) { + public static boolean isSecurityEnabled(final Configuration config) { Validate.notNull(config); return "kerberos".equalsIgnoreCase(config.get("hadoop.security.authentication")); } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java index cd9683bc7cc7..9efc0f6ea583 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java @@ -116,6 +116,9 @@ public String toString() { // Hadoop Configuration, Filesystem, and UserGroupInformation (optional) private final AtomicReference hdfsResources = new AtomicReference<>(); + // Holder of cached Configuration information so validation does not reload the same config over and over + private final AtomicReference validationResourceHolder = new AtomicReference<>(); + @Override protected void init(ProcessorInitializationContext context) { hdfsResources.set(new HdfsResources(null, null, null)); @@ -147,12 +150,21 @@ protected Collection customValidate(ValidationContext validati final List results = new ArrayList<>(); if (!StringUtils.isBlank(configResources)) { - Configuration conf = null; try { - conf = getConfigurationFromResources(configResources); + ValidationResources resources = validationResourceHolder.get(); + + // if no resources in the holder, or if the holder has different resources loaded, + // then load the Configuration and set the new resources in the holder + if (resources == null || !configResources.equals(resources.getConfigResources())) { + getLogger().debug("Reloading validation resources"); + resources = new ValidationResources(configResources, getConfigurationFromResources(configResources)); + validationResourceHolder.set(resources); + } + final Configuration conf = resources.getConfiguration(); results.addAll(KerberosProperties.validatePrincipalAndKeytab( this.getClass().getSimpleName(), conf, principal, keytab, getLogger())); + } catch (IOException e) { results.add(new ValidationResult.Builder() .valid(false) @@ -452,4 +464,23 @@ public UserGroupInformation getUserGroupInformation() { return userGroupInformation; } } + + static protected class ValidationResources { + private final String configResources; + private final Configuration configuration; + + public ValidationResources(String configResources, Configuration configuration) { + this.configResources = configResources; + this.configuration = configuration; + } + + public String getConfigResources() { + return configResources; + } + + public Configuration getConfiguration() { + return configuration; + } + } + } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java index 346513543a36..1791cfee0062 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java @@ -65,6 +65,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; @Tags({ "hbase", "client"}) @CapabilityDescription("Implementation of HBaseClientService for HBase 1.1.2. This service can be configured by providing " + @@ -90,6 +91,9 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme private List properties; private KerberosProperties kerberosProperties; + // Holder of cached Configuration information so validation does not reload the same config over and over + private final AtomicReference validationResourceHolder = new AtomicReference<>(); + @Override protected void init(ControllerServiceInitializationContext config) throws InitializationException { this.kerberosProperties = getKerberosProperties(); @@ -145,7 +149,17 @@ protected Collection customValidate(ValidationContext validati if (confFileProvided) { final String configFiles = validationContext.getProperty(HADOOP_CONF_FILES).getValue(); - final Configuration hbaseConfig = getConfigurationFromFiles(configFiles); + ValidationResources resources = validationResourceHolder.get(); + + // if no resources in the holder, or if the holder has different resources loaded, + // then load the Configuration and set the new resources in the holder + if (resources == null || !configFiles.equals(resources.getConfigResources())) { + getLogger().debug("Reloading validation resources"); + resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles)); + validationResourceHolder.set(resources); + } + + final Configuration hbaseConfig = resources.getConfiguration(); final String principal = validationContext.getProperty(kerberosProperties.getKerberosPrincipal()).getValue(); final String keytab = validationContext.getProperty(kerberosProperties.getKerberosKeytab()).getValue(); @@ -372,4 +386,23 @@ protected ResultScanner getResults(final Table table, final Collection c return table.getScanner(scan); } + + static protected class ValidationResources { + private final String configResources; + private final Configuration configuration; + + public ValidationResources(String configResources, Configuration configuration) { + this.configResources = configResources; + this.configuration = configuration; + } + + public String getConfigResources() { + return configResources; + } + + public Configuration getConfiguration() { + return configuration; + } + } + }