From 33d9d97f2866c15c703c6e654cc89478c5c1838a Mon Sep 17 00:00:00 2001 From: merrimanr Date: Fri, 14 Jul 2017 14:08:02 -0500 Subject: [PATCH 1/2] initial commit --- .../METRON/CURRENT/package/templates/metron.j2 | 2 +- .../metron-rest/src/main/config/rest_application.yml | 4 ---- .../java/org/apache/metron/rest/MetronRestConstants.java | 1 - .../java/org/apache/metron/rest/config/HadoopConfig.java | 5 +++-- metron-interface/metron-rest/src/main/scripts/metron-rest | 5 ++++- .../org/apache/metron/rest/config/HadoopConfigTest.java | 6 ------ 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 7543a67d53..e45370ada3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -30,7 +30,7 @@ METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}" METRON_SPRING_OPTIONS="{{metron_spring_options}}" ZOOKEEPER="{{zookeeper_quorum}}" BROKERLIST="{{kafka_brokers}}" -HDFS_URL="{{default_fs}}" +HADOOP_CONF_DIR="/etc/hadoop/conf/" STORM_REST_URL="{{storm_rest_addr}}" SECURITY_ENABLED={{security_enabled|lower}} {% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}" diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index dbb0b43185..288b59839e 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -29,10 +29,6 @@ kafka: broker: url: ${BROKERLIST} -hdfs: - namenode: - url: ${HDFS_URL} - grok: path: temp: ${METRON_TEMP_GROK_PATH} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 2a147e64b3..7d55c99cc0 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -51,7 +51,6 @@ public class MetronRestConstants { public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url"; public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url"; - public static final String DEFAULT_HDFS_URL = "file:///"; public static final String KERBEROS_ENABLED_SPRING_PROPERTY = "kerberos.enabled"; public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY = "kerberos.principal"; diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java index 7b1bf2f3a5..3ce6592774 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java @@ -39,9 +39,10 @@ public HadoopConfig(Environment environment) { @Bean public org.apache.hadoop.conf.Configuration configuration() throws IOException { org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration(); - configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)); + if (environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY) != null) { + configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY)); + } if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) { - configuration.set("hadoop.security.authentication", "KERBEROS"); UserGroupInformation.setConfiguration(configuration); String keyTabLocation = environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY); String userPrincipal = environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY); diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest index 3fa1df01ed..99fc4ac407 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest @@ -44,6 +44,8 @@ fi PIDFILE="$METRON_PID_DIR/$NAME.pid" LOCKFILE=/var/lock/subsys/$NAME +LOADER_PATH="$HADOOP_CONF_DIR" + # the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml if [[ !($METRON_SPRING_PROFILES_ACTIVE == *"vagrant"*) ]]; then METRON_SPRING_OPTIONS+=" --spring.config.location=$METRON_HOME/config/rest_application.yml" @@ -56,8 +58,9 @@ if [ $METRON_JDBC_PASSWORD ]; then METRON_SPRING_OPTIONS+=" --spring.datasource.password=$METRON_JDBC_PASSWORD" fi if [ $METRON_JDBC_CLIENT_PATH ]; then - METRON_JVMFLAGS+=" -Dloader.path=$METRON_JDBC_CLIENT_PATH" + LOADER_PATH+=",$METRON_JDBC_CLIENT_PATH" fi +METRON_JVMFLAGS+=" -Dloader.path=$LOADER_PATH" DAEMON="java $METRON_JVMFLAGS -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar $METRON_SPRING_OPTIONS" # diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java index c262783f9a..7504e3ed1f 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java @@ -56,7 +56,6 @@ public void setUp() throws Exception { @Test public void configurationShouldReturnProperKerberosConfiguration() throws IOException { - when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem"); when(environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY)).thenReturn("metron keytabLocation"); when(environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY)).thenReturn("metron principal"); @@ -67,14 +66,10 @@ public void configurationShouldReturnProperKerberosConfiguration() throws IOExce verifyStatic(); UserGroupInformation.setConfiguration(any(Configuration.class)); UserGroupInformation.loginUserFromKeytab("metron keytabLocation", "metron principal"); - - assertEquals("default filesystem", configuration.get("fs.defaultFS")); - assertEquals("KERBEROS", configuration.get("hadoop.security.authentication")); } @Test public void configurationShouldReturnProperConfiguration() throws IOException { - when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem"); when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(false); Configuration configuration = hadoopConfig.configuration(); @@ -83,7 +78,6 @@ public void configurationShouldReturnProperConfiguration() throws IOException { UserGroupInformation.setConfiguration(any(Configuration.class)); UserGroupInformation.loginUserFromKeytab(anyString(), anyString()); - assertEquals("default filesystem", configuration.get("fs.defaultFS")); assertEquals("simple", configuration.get("hadoop.security.authentication")); } } From 71dc342f9e39ce6e54e5252a5fb2991d83c757e8 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 26 Jul 2017 13:44:17 -0500 Subject: [PATCH 2/2] removed hdfs url property --- .../main/java/org/apache/metron/rest/MetronRestConstants.java | 4 ---- .../main/java/org/apache/metron/rest/config/HadoopConfig.java | 3 --- 2 files changed, 7 deletions(-) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 6c983a9ac2..a98760b6ba 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -30,8 +30,6 @@ public class MetronRestConstants { public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp"; public static final String GROK_CLASS_NAME = GrokParser.class.getName(); public static final String GROK_PATH_KEY = "grokPath"; - public static final String GROK_STATEMENT_KEY = "grokStatement"; - public static final String GROK_PATTERN_LABEL_KEY = "patternLabel"; public static final String STORM_UI_SPRING_PROPERTY = "storm.ui.url"; public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary"; @@ -52,8 +50,6 @@ public class MetronRestConstants { public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url"; - public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url"; - public static final String KERBEROS_ENABLED_SPRING_PROPERTY = "kerberos.enabled"; public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY = "kerberos.principal"; public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab"; diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java index 3ce6592774..98c8108f45 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java @@ -39,9 +39,6 @@ public HadoopConfig(Environment environment) { @Bean public org.apache.hadoop.conf.Configuration configuration() throws IOException { org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration(); - if (environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY) != null) { - configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY)); - } if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) { UserGroupInformation.setConfiguration(configuration); String keyTabLocation = environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY);