From 754a1ba7b579d4507aa728d6df17c535130c80d4 Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Tue, 1 Nov 2016 16:36:33 -0400 Subject: [PATCH 1/7] working on js parser --- metron-platform/metron-parsers/pom.xml | 5 + .../org/apache/metron/parsers/JSParser.java | 132 ++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 3e42b35c46..57de42d022 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -184,6 +184,11 @@ 2.2.6 test + + io.vertx + lang-nashorn + 1.0.0-beta2 + diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java new file mode 100644 index 0000000000..072de67655 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java @@ -0,0 +1,132 @@ +package org.apache.metron.parsers; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.script.Invocable; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.parsers.interfaces.MessageParser; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JSParser implements MessageParser,Serializable{ + + protected static final Logger LOG = LoggerFactory.getLogger(JSParser.class); + protected String jsPath; + protected ScriptEngine engine; + protected String parseFunction; + //protected String language; + protected String commonJS="/scripts/js/common"; + + @Override + public void configure(Map config) { + // TODO Auto-generated method stub + this.jsPath=(String) config.get("path"); + this.parseFunction=(String)config.get("function"); + if(this.parseFunction==null) + this.parseFunction="parse"; + } + + public InputStream openInputStream(String streamName) throws IOException { + FileSystem fs = FileSystem.get(new Configuration()); + Path path = new Path(streamName); + if(fs.exists(path)) { + return fs.open(path); + } else { + return getClass().getResourceAsStream(streamName); + } + } + + @Override + public void init() { + // TODO Auto-generated method stub + engine = new ScriptEngineManager().getEngineByName("nashorn"); + try{ + InputStream commonStream = openInputStream(this.commonJS); + if (commonStream == null) { + throw new RuntimeException( + "Unable to initialize JS Parser: Unable to load " + this.commonJS + " from either classpath or HDFS"); + } + + engine.eval(new InputStreamReader(commonStream)); + if (LOG.isDebugEnabled()) { + LOG.debug("Loading parser-specific patterns from: " + this.jsPath); + } + + InputStream patterInputStream = openInputStream(this.jsPath); + if (patterInputStream == null) { + throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + this.jsPath + + " from either classpath or HDFS"); + } + engine.eval(new InputStreamReader(patterInputStream)); + }catch(Throwable e){ + LOG.error(e.getMessage(), e); + throw new RuntimeException("JS parser Error: " + e.getMessage(), e); + } + + Invocable invocable = (Invocable) engine; + + } + + @Override + public List parse(byte[] rawMessage) { + // TODO Auto-generated method stub + List messages = new ArrayList(); + if(engine==null) + init(); + String originalMessage = null; + try{ + originalMessage = new String(rawMessage, "UTF-8"); + if (LOG.isDebugEnabled()) { + LOG.debug("JS parser parsing message: " + originalMessage); + } + Invocable invocable = (Invocable) engine; + + Object result = invocable.invokeFunction(this.parseFunction,originalMessage); + JSONObject jsonObject = new JSONObject((Map)result); + messages.add(jsonObject); + return messages; + }catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IllegalStateException("Grok parser Error: " + e.getMessage() + " on " + originalMessage , e); + } + } + + @Override + public boolean validate(JSONObject message) { + // TODO Auto-generated method stub + return false; + } + + public static void main(String[] args){ + ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn"); + String json ="var test = {'id': 10,'Hello': 'World','test':" + + " {'Lorem' : 'Ipsum','java' : true },}"; + try{ + engine.eval("var fun1 = function() {\n"+json+";return test;\n};"); + Invocable invocable = (Invocable) engine; + + Object result = invocable.invokeFunction("fun1", "Peter Parker"); + System.out.println(result); + Map obj = (Map)result; + System.out.println(obj); + JSONObject jsonObject = new JSONObject(obj); + System.out.println(jsonObject); + }catch(Exception ex){ + ex.printStackTrace(); + } + + } + +} From f4d0cec86cbf3151791dad660e2e13d64007b71e Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Wed, 2 Nov 2016 10:24:58 -0400 Subject: [PATCH 2/7] renamed JSParser to ScriptPArser --- metron-platform/metron-parsers/pom.xml | 8 ++-- .../{JSParser.java => ScriptParser.java} | 45 +++++++++++-------- 2 files changed, 31 insertions(+), 22 deletions(-) rename metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/{JSParser.java => ScriptParser.java} (65%) diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 57de42d022..30ea0d09ac 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -184,10 +184,10 @@ 2.2.6 test - - io.vertx - lang-nashorn - 1.0.0-beta2 + + org.python + jython-standalone + 2.7.0 diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java similarity index 65% rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java rename to metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java index 072de67655..40670cc5d2 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/JSParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java @@ -10,6 +10,7 @@ import javax.script.Invocable; import javax.script.ScriptEngine; +import javax.script.ScriptEngineFactory; import javax.script.ScriptEngineManager; import org.apache.hadoop.conf.Configuration; @@ -20,24 +21,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class JSParser implements MessageParser,Serializable{ +public class ScriptParser implements MessageParser,Serializable{ - protected static final Logger LOG = LoggerFactory.getLogger(JSParser.class); - protected String jsPath; + protected static final Logger LOG = LoggerFactory.getLogger(ScriptParser.class); + protected String scriptPath; protected ScriptEngine engine; protected String parseFunction; - //protected String language; - protected String commonJS="/scripts/js/common"; + protected String language; + protected String commonScript="/scripts/"; @Override public void configure(Map config) { // TODO Auto-generated method stub - this.jsPath=(String) config.get("path"); + this.scriptPath=(String) config.get("path"); this.parseFunction=(String)config.get("function"); + this.language=(String)config.get("language"); + this.commonScript=this.commonScript+language+"/common"; if(this.parseFunction==null) this.parseFunction="parse"; } - + //Should this be sent to the interface as a default method? public InputStream openInputStream(String streamName) throws IOException { FileSystem fs = FileSystem.get(new Configuration()); Path path = new Path(streamName); @@ -51,28 +54,28 @@ public InputStream openInputStream(String streamName) throws IOException { @Override public void init() { // TODO Auto-generated method stub - engine = new ScriptEngineManager().getEngineByName("nashorn"); + engine = new ScriptEngineManager().getEngineByName("js"); try{ - InputStream commonStream = openInputStream(this.commonJS); + InputStream commonStream = openInputStream(this.commonScript); if (commonStream == null) { throw new RuntimeException( - "Unable to initialize JS Parser: Unable to load " + this.commonJS + " from either classpath or HDFS"); + "Unable to initialize "+this.language+" Parser: Unable to load " + this.commonScript + " from either classpath or HDFS"); } engine.eval(new InputStreamReader(commonStream)); if (LOG.isDebugEnabled()) { - LOG.debug("Loading parser-specific patterns from: " + this.jsPath); + LOG.debug("Loading parser-specific functions from: " + this.scriptPath); } - InputStream patterInputStream = openInputStream(this.jsPath); + InputStream patterInputStream = openInputStream(this.scriptPath); if (patterInputStream == null) { - throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + this.jsPath + throw new RuntimeException("Script parser unable to initialize "+this.language+" parser: Unable to load " + this.scriptPath + " from either classpath or HDFS"); } engine.eval(new InputStreamReader(patterInputStream)); }catch(Throwable e){ LOG.error(e.getMessage(), e); - throw new RuntimeException("JS parser Error: " + e.getMessage(), e); + throw new RuntimeException(this.language+" Script parser Error: " + e.getMessage(), e); } Invocable invocable = (Invocable) engine; @@ -110,11 +113,17 @@ public boolean validate(JSONObject message) { } public static void main(String[] args){ - ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn"); - String json ="var test = {'id': 10,'Hello': 'World','test':" - + " {'Lorem' : 'Ipsum','java' : true },}"; + ScriptEngine engine = new ScriptEngineManager().getEngineByName("python"); + /*for(ScriptEngineFactory sf: new ScriptEngineManager().getEngineFactories()){ + if("jython".equals(sf.getEngineName())){ + engine = sf.getScriptEngine(); + break; + } + }*/ + String json ="\ttest = '{\"id\": 10,\"Hello\": \"World\"}'"; + System.out.println("import json\ndef fun1(name):\n"+json+"\n\tprint test\n\treturn json.loads(test)\n");; try{ - engine.eval("var fun1 = function() {\n"+json+";return test;\n};"); + engine.eval("import json\ndef fun1(name):\n"+json+"\n\tprint test\n\treturn json.loads(test)\n"); Invocable invocable = (Invocable) engine; Object result = invocable.invokeFunction("fun1", "Peter Parker"); From 91bf93d436d95877f2633518ca5393510f590d11 Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Wed, 2 Nov 2016 14:44:34 -0400 Subject: [PATCH 3/7] script parsing support for js,python and groovy. --- .../metron-integration-test/.pydevproject | 5 + .../src/main/sample/scripts/test.groovy | 13 +++ .../src/main/sample/scripts/test.js | 13 +++ .../src/main/sample/scripts/test.py | 12 +++ metron-platform/metron-parsers/pom.xml | 5 + .../apache/metron/parsers/ScriptParser.java | 92 +++++++++++++------ .../src/main/resources/scripts/groovy/common | 5 + .../src/main/resources/scripts/js/common | 3 + .../src/main/resources/scripts/python/common | 4 + .../metron/parsers/GroovyParserTest.java | 81 ++++++++++++++++ .../apache/metron/parsers/JSParserTest.java | 81 ++++++++++++++++ .../metron/parsers/PythonParserTest.java | 81 ++++++++++++++++ .../metron/parsers/ScriptParserTest.java | 85 +++++++++++++++++ 13 files changed, 451 insertions(+), 29 deletions(-) create mode 100644 metron-platform/metron-integration-test/.pydevproject create mode 100644 metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy create mode 100644 metron-platform/metron-integration-test/src/main/sample/scripts/test.js create mode 100644 metron-platform/metron-integration-test/src/main/sample/scripts/test.py create mode 100644 metron-platform/metron-parsers/src/main/resources/scripts/groovy/common create mode 100644 metron-platform/metron-parsers/src/main/resources/scripts/js/common create mode 100644 metron-platform/metron-parsers/src/main/resources/scripts/python/common create mode 100644 metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java create mode 100644 metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java create mode 100644 metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java create mode 100644 metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java diff --git a/metron-platform/metron-integration-test/.pydevproject b/metron-platform/metron-integration-test/.pydevproject new file mode 100644 index 0000000000..40e9f40a0a --- /dev/null +++ b/metron-platform/metron-integration-test/.pydevproject @@ -0,0 +1,5 @@ + + +Default +python 2.7 + diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy b/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy new file mode 100644 index 0000000000..44894fb92a --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy @@ -0,0 +1,13 @@ +def parse(rawMessage){ + String[] parts=rawMessage.split("\\|"); + def message = MetronMessage(); + message["customerId"]=parts[1]; + message["first_name"]=parts[2]; + message["last_name"]=parts[3]; + message["age"]=parts[4]; + message["login-time"]=parts[0]; + message["ip-address"]=parts[5]; + message["os"]=parts[6]; + message["device"]=parts[7]; + return message; + } \ No newline at end of file diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.js b/metron-platform/metron-integration-test/src/main/sample/scripts/test.js new file mode 100644 index 0000000000..3f409e4881 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.js @@ -0,0 +1,13 @@ +var parse =function(rawMessage){ + parts=rawMessage.split("|"); + var message = new MetronMessage("userlog"); + message["customerId"]=parts[1]; + message["first_name"]=parts[2]; + message["last_name"]=parts[3]; + message["age"]=parts[4]; + message["login-time"]=parts[0]; + message["ip-address"]=parts[5]; + message["os"]=parts[6]; + message["device"]=parts[7]; + return message; +}; \ No newline at end of file diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.py b/metron-platform/metron-integration-test/src/main/sample/scripts/test.py new file mode 100644 index 0000000000..80e37681a5 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.py @@ -0,0 +1,12 @@ +def parse(rawMessage): + parts=rawMessage.split("|"); + message = MetronMessage("userlog") + message["customerId"]=parts[1]; + message["first_name"]=parts[2]; + message["last_name"]=parts[3]; + message["age"]=parts[4]; + message["login-time"]=parts[0]; + message["ip-address"]=parts[5]; + message["os"]=parts[6]; + message["device"]=parts[7]; + return message; diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 30ea0d09ac..b202c0c737 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -189,6 +189,11 @@ jython-standalone 2.7.0 + + org.codehaus.groovy + groovy-all + 2.4.7 + diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java index 40670cc5d2..f427fd2b6c 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java @@ -4,7 +4,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; @@ -16,11 +19,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.metron.common.Constants; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; + public class ScriptParser implements MessageParser,Serializable{ protected static final Logger LOG = LoggerFactory.getLogger(ScriptParser.class); @@ -29,6 +36,9 @@ public class ScriptParser implements MessageParser,Serializable{ protected String parseFunction; protected String language; protected String commonScript="/scripts/"; + protected List timeFields = new ArrayList<>(); + protected String timestampField; + protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); @Override public void configure(Map config) { @@ -54,7 +64,7 @@ public InputStream openInputStream(String streamName) throws IOException { @Override public void init() { // TODO Auto-generated method stub - engine = new ScriptEngineManager().getEngineByName("js"); + engine = new ScriptEngineManager().getEngineByName(this.language); try{ InputStream commonStream = openInputStream(this.commonScript); if (commonStream == null) { @@ -97,45 +107,69 @@ public List parse(byte[] rawMessage) { Invocable invocable = (Invocable) engine; Object result = invocable.invokeFunction(this.parseFunction,originalMessage); - JSONObject jsonObject = new JSONObject((Map)result); - messages.add(jsonObject); - return messages; + JSONObject message = new JSONObject((Map)result); + + if (message.size() == 0) + throw new RuntimeException("Script produced a null message. Original message was: " + + originalMessage + " and the parsed message was: " + message + " . Check the function at: " + + this.scriptPath); + + message.put("original_string", originalMessage); + for (String timeField : timeFields) { + String fieldValue = (String) message.get(timeField); + if (fieldValue != null) { + message.put(timeField, toEpoch(fieldValue)); + } + } + if (timestampField != null) { + message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); + } + messages.add(message); + if (LOG.isDebugEnabled()) { + LOG.debug("Grok parser parsed message: " + message); + } }catch (Exception e) { LOG.error(e.getMessage(), e); throw new IllegalStateException("Grok parser Error: " + e.getMessage() + " on " + originalMessage , e); } + return messages; } @Override public boolean validate(JSONObject message) { // TODO Auto-generated method stub - return false; + return true; } - public static void main(String[] args){ - ScriptEngine engine = new ScriptEngineManager().getEngineByName("python"); - /*for(ScriptEngineFactory sf: new ScriptEngineManager().getEngineFactories()){ - if("jython".equals(sf.getEngineName())){ - engine = sf.getScriptEngine(); - break; - } - }*/ - String json ="\ttest = '{\"id\": 10,\"Hello\": \"World\"}'"; - System.out.println("import json\ndef fun1(name):\n"+json+"\n\tprint test\n\treturn json.loads(test)\n");; - try{ - engine.eval("import json\ndef fun1(name):\n"+json+"\n\tprint test\n\treturn json.loads(test)\n"); - Invocable invocable = (Invocable) engine; + protected long toEpoch(String datetime) throws ParseException { - Object result = invocable.invokeFunction("fun1", "Peter Parker"); - System.out.println(result); - Map obj = (Map)result; - System.out.println(obj); - JSONObject jsonObject = new JSONObject(obj); - System.out.println(jsonObject); - }catch(Exception ex){ - ex.printStackTrace(); - } - - } + LOG.debug("Grok parser converting timestamp to epoch: {}", datetime); + LOG.debug("Grok parser's DateFormat has TimeZone: {}", dateFormat.getTimeZone()); + + Date date = dateFormat.parse(datetime); + if (LOG.isDebugEnabled()) { + LOG.debug("Grok parser converted timestamp to epoch: " + date); + } + return date.getTime(); + } + + protected long formatTimestamp(Object value) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Grok parser formatting timestamp" + value); + } + + + if (value == null) { + throw new RuntimeException(this.parseFunction + " parser does not include field " + timestampField); + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } else { + return Long.parseLong(Joiner.on("").join(Splitter.on('.').split(value + ""))); + } + } + + } diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common b/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common new file mode 100644 index 0000000000..132495d81e --- /dev/null +++ b/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common @@ -0,0 +1,5 @@ +def MetronMessage(){ + def message=[:]; + message["source"]="userlog"; + return message; +} \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/js/common b/metron-platform/metron-parsers/src/main/resources/scripts/js/common new file mode 100644 index 0000000000..2cf2075fc5 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/resources/scripts/js/common @@ -0,0 +1,3 @@ +function MetronMessage(name) { + this.source=name +} \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/python/common b/metron-platform/metron-parsers/src/main/resources/scripts/python/common new file mode 100644 index 0000000000..03878271da --- /dev/null +++ b/metron-platform/metron-parsers/src/main/resources/scripts/python/common @@ -0,0 +1,4 @@ +def MetronMessage(name): + message={} + message["source"]="userlog" + return message \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java new file mode 100644 index 0000000000..d89856ca1a --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java @@ -0,0 +1,81 @@ +package org.apache.metron.parsers; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.adrianwalker.multilinestring.Multiline; + +public class GroovyParserTest extends ScriptParserTest { + + /** + { + "customerId":117, + "first_name":"karthik", + "last_name":"narayanan", + "age":"38", + "login-time":"2016-01-28 15:29:48", + "ip-address":"216.21.170.221", + "os":"windows 10", + "device":"Dell Inspiron" + } + */ + @Multiline + public String result=" {"+ + "\"source\":\"userlog\","+ + "\"customerId\":117,"+ + "\"first_name\":\"karthik\","+ + "\"last_name\":\"narayanan\","+ + "\"age\":\"38\","+ + "\"login-time\":\"2016-01-28 15:29:48\","+ + "\"ip-address\":\"216.21.170.221\","+ + "\"os\":\"windows 10\","+ + "\"device\":\"Dell Inspiron\""+ + "\"original_string\":\"2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron\""+ + "}"; + + @Override + public Map getTestData() { + Map testData = new HashMap(); + String input = "2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron"; + testData.put(input,result); + return testData; + } + + @Override + public String getScriptPath() { + // TODO Auto-generated method stub + return "../metron-integration-test/src/main/sample/scripts/test.groovy"; + } + + @Override + public String getParseFunction() { + // TODO Auto-generated method stub + return "parse"; + } + + @Override + public String getLanguage() { + // TODO Auto-generated method stub + return "groovy"; + } + + @Override + public List getTimeFields() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getDateFormat() { + // TODO Auto-generated method stub + return "yyyy-MM-dd HH:mm:ss"; + } + + @Override + public String getTimestampField() { + // TODO Auto-generated method stub + return "login-time"; + } + +} diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java new file mode 100644 index 0000000000..92f5e7ba4f --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java @@ -0,0 +1,81 @@ +package org.apache.metron.parsers; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.adrianwalker.multilinestring.Multiline; + +public class JSParserTest extends ScriptParserTest { + + /** + { + "customerId":117, + "first_name":"karthik", + "last_name":"narayanan", + "age":"38", + "login-time":"2016-01-28 15:29:48", + "ip-address":"216.21.170.221", + "os":"windows 10", + "device":"Dell Inspiron" + } + */ + @Multiline + public String result=" {"+ + "\"source\":\"userlog\","+ + "\"customerId\":117,"+ + "\"first_name\":\"karthik\","+ + "\"last_name\":\"narayanan\","+ + "\"age\":\"38\","+ + "\"login-time\":\"2016-01-28 15:29:48\","+ + "\"ip-address\":\"216.21.170.221\","+ + "\"os\":\"windows 10\","+ + "\"device\":\"Dell Inspiron\""+ + "\"original_string\":\"2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron\""+ + "}"; + + @Override + public Map getTestData() { + Map testData = new HashMap(); + String input = "2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron"; + testData.put(input,result); + return testData; + } + + @Override + public String getScriptPath() { + // TODO Auto-generated method stub + return "../metron-integration-test/src/main/sample/scripts/test.js"; + } + + @Override + public String getParseFunction() { + // TODO Auto-generated method stub + return "parse"; + } + + @Override + public String getLanguage() { + // TODO Auto-generated method stub + return "js"; + } + + @Override + public List getTimeFields() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getDateFormat() { + // TODO Auto-generated method stub + return "yyyy-MM-dd HH:mm:ss"; + } + + @Override + public String getTimestampField() { + // TODO Auto-generated method stub + return "login_time"; + } + +} diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java new file mode 100644 index 0000000000..2bd1a82ffc --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java @@ -0,0 +1,81 @@ +package org.apache.metron.parsers; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.adrianwalker.multilinestring.Multiline; + +public class PythonParserTest extends ScriptParserTest { + + /** + { + "customerId":117, + "first_name":"karthik", + "last_name":"narayanan", + "age":"38", + "login-time":"2016-01-28 15:29:48", + "ip-address":"216.21.170.221", + "os":"windows 10", + "device":"Dell Inspiron" + } + */ + @Multiline + public String result=" {"+ + "\"source\":\"userlog\","+ + "\"customerId\":117,"+ + "\"first_name\":\"karthik\","+ + "\"last_name\":\"narayanan\","+ + "\"age\":\"38\","+ + "\"login-time\":\"2016-01-28 15:29:48\","+ + "\"ip-address\":\"216.21.170.221\","+ + "\"os\":\"windows 10\","+ + "\"device\":\"Dell Inspiron\""+ + "\"original_string\":\"2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron\""+ + "}"; + + @Override + public Map getTestData() { + Map testData = new HashMap(); + String input = "2016-01-28 15:29:48|117|karthik|narayanan|38|216.21.170.221|windows 10|Dell Inspiron"; + testData.put(input,result); + return testData; + } + + @Override + public String getScriptPath() { + // TODO Auto-generated method stub + return "../metron-integration-test/src/main/sample/scripts/test.py"; + } + + @Override + public String getParseFunction() { + // TODO Auto-generated method stub + return "parse"; + } + + @Override + public String getLanguage() { + // TODO Auto-generated method stub + return "python"; + } + + @Override + public List getTimeFields() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getDateFormat() { + // TODO Auto-generated method stub + return "yyyy-MM-dd HH:mm:ss"; + } + + @Override + public String getTimestampField() { + // TODO Auto-generated method stub + return "login_time"; + } + +} diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java new file mode 100644 index 0000000000..8b54c75384 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java @@ -0,0 +1,85 @@ +package org.apache.metron.parsers; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.adrianwalker.multilinestring.Multiline; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; + +import junit.framework.Assert; + +public abstract class ScriptParserTest { + + public abstract Map getTestData(); + public abstract String getScriptPath(); + public abstract String getParseFunction(); + public abstract String getLanguage(); + public abstract List getTimeFields(); + public abstract String getDateFormat(); + public abstract String getTimestampField(); + + @Test + public void test() throws IOException, ParseException { + + Map parserConfig = new HashMap<>(); + parserConfig.put("path", getScriptPath()); + parserConfig.put("function", getParseFunction()); + parserConfig.put("language", getLanguage()); + parserConfig.put("timeStampField",getTimestampField()); + parserConfig.put("dateFormat", getDateFormat()); + parserConfig.put("timeFields", getTimeFields()); + + ScriptParser scriptParser = new ScriptParser(); + scriptParser.configure(parserConfig); + scriptParser.init(); + + JSONParser jsonParser = new JSONParser(); + Map testData = getTestData(); + for( Map.Entry e : testData.entrySet() ){ + + JSONObject expected = (JSONObject) jsonParser.parse(e.getValue()); + byte[] rawMessage = e.getKey().getBytes(); + + List parsedList = scriptParser.parse(rawMessage); + System.out.println(parsedList.get(0)); + Assert.assertEquals(1, parsedList.size()); + compare(expected, parsedList.get(0)); + } + + } + + public boolean compare(JSONObject expected, JSONObject actual) { + MapDifference mapDifferences = Maps.difference(expected, actual); + if (mapDifferences.entriesOnlyOnLeft().size() > 0) Assert.fail("Expected JSON has extra parameters: " + mapDifferences.entriesOnlyOnLeft()); + if (mapDifferences.entriesOnlyOnRight().size() > 0) Assert.fail("Actual JSON has extra parameters: " + mapDifferences.entriesOnlyOnRight()); + Map actualDifferences = new HashMap(); + if (mapDifferences.entriesDiffering().size() > 0) { + Map differences = Collections.unmodifiableMap(mapDifferences.entriesDiffering()); + for (Object key : differences.keySet()) { + Object expectedValueObject = expected.get(key); + Object actualValueObject = actual.get(key); + if (expectedValueObject instanceof Long || expectedValueObject instanceof Integer) { + Long expectedValue = Long.parseLong(expectedValueObject.toString()); + Long actualValue = Long.parseLong(actualValueObject.toString()); + if (!expectedValue.equals(actualValue)) { + actualDifferences.put(key, differences.get(key)); + } + } else { + actualDifferences.put(key, differences.get(key)); + } + } + } + if (actualDifferences.size() > 0) Assert.fail("Expected and Actual JSON values don't match: " + actualDifferences); + return true; + } + +} From d1af778f9e61fefbf82942235ee45ed084537910 Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Wed, 2 Nov 2016 16:06:19 -0400 Subject: [PATCH 4/7] removed TODOs --- .../metron-integration-test/.pydevproject | 5 ---- .../apache/metron/parsers/ScriptParser.java | 18 ++++++------- .../metron/parsers/GroovyParserTest.java | 23 ++--------------- .../apache/metron/parsers/JSParserTest.java | 25 +++---------------- .../metron/parsers/PythonParserTest.java | 23 ++--------------- 5 files changed, 14 insertions(+), 80 deletions(-) delete mode 100644 metron-platform/metron-integration-test/.pydevproject diff --git a/metron-platform/metron-integration-test/.pydevproject b/metron-platform/metron-integration-test/.pydevproject deleted file mode 100644 index 40e9f40a0a..0000000000 --- a/metron-platform/metron-integration-test/.pydevproject +++ /dev/null @@ -1,5 +0,0 @@ - - -Default -python 2.7 - diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java index f427fd2b6c..7d6e7a57a7 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java @@ -42,7 +42,6 @@ public class ScriptParser implements MessageParser,Serializable{ @Override public void configure(Map config) { - // TODO Auto-generated method stub this.scriptPath=(String) config.get("path"); this.parseFunction=(String)config.get("function"); this.language=(String)config.get("language"); @@ -63,7 +62,6 @@ public InputStream openInputStream(String streamName) throws IOException { @Override public void init() { - // TODO Auto-generated method stub engine = new ScriptEngineManager().getEngineByName(this.language); try{ InputStream commonStream = openInputStream(this.commonScript); @@ -94,7 +92,6 @@ public void init() { @Override public List parse(byte[] rawMessage) { - // TODO Auto-generated method stub List messages = new ArrayList(); if(engine==null) init(); @@ -102,7 +99,7 @@ public List parse(byte[] rawMessage) { try{ originalMessage = new String(rawMessage, "UTF-8"); if (LOG.isDebugEnabled()) { - LOG.debug("JS parser parsing message: " + originalMessage); + LOG.debug("Script parser parsing message: " + originalMessage); } Invocable invocable = (Invocable) engine; @@ -126,29 +123,28 @@ public List parse(byte[] rawMessage) { } messages.add(message); if (LOG.isDebugEnabled()) { - LOG.debug("Grok parser parsed message: " + message); + LOG.debug("Script parser parsed message: " + message); } }catch (Exception e) { LOG.error(e.getMessage(), e); - throw new IllegalStateException("Grok parser Error: " + e.getMessage() + " on " + originalMessage , e); + throw new IllegalStateException("Script parser Error: " + e.getMessage() + " on " + originalMessage , e); } return messages; } @Override public boolean validate(JSONObject message) { - // TODO Auto-generated method stub return true; } protected long toEpoch(String datetime) throws ParseException { - LOG.debug("Grok parser converting timestamp to epoch: {}", datetime); - LOG.debug("Grok parser's DateFormat has TimeZone: {}", dateFormat.getTimeZone()); + LOG.debug("Script parser converting timestamp to epoch: {}", datetime); + LOG.debug("Script parser's DateFormat has TimeZone: {}", dateFormat.getTimeZone()); Date date = dateFormat.parse(datetime); if (LOG.isDebugEnabled()) { - LOG.debug("Grok parser converted timestamp to epoch: " + date); + LOG.debug("Script parser converted timestamp to epoch: " + date); } return date.getTime(); @@ -157,7 +153,7 @@ protected long toEpoch(String datetime) throws ParseException { protected long formatTimestamp(Object value) { if (LOG.isDebugEnabled()) { - LOG.debug("Grok parser formatting timestamp" + value); + LOG.debug("Script parser formatting timestamp" + value); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java index d89856ca1a..59183b3a56 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java @@ -7,21 +7,8 @@ import org.adrianwalker.multilinestring.Multiline; public class GroovyParserTest extends ScriptParserTest { - - /** - { - "customerId":117, - "first_name":"karthik", - "last_name":"narayanan", - "age":"38", - "login-time":"2016-01-28 15:29:48", - "ip-address":"216.21.170.221", - "os":"windows 10", - "device":"Dell Inspiron" - } - */ - @Multiline - public String result=" {"+ + + public String result=" {"+ "\"source\":\"userlog\","+ "\"customerId\":117,"+ "\"first_name\":\"karthik\","+ @@ -44,37 +31,31 @@ public Map getTestData() { @Override public String getScriptPath() { - // TODO Auto-generated method stub return "../metron-integration-test/src/main/sample/scripts/test.groovy"; } @Override public String getParseFunction() { - // TODO Auto-generated method stub return "parse"; } @Override public String getLanguage() { - // TODO Auto-generated method stub return "groovy"; } @Override public List getTimeFields() { - // TODO Auto-generated method stub return null; } @Override public String getDateFormat() { - // TODO Auto-generated method stub return "yyyy-MM-dd HH:mm:ss"; } @Override public String getTimestampField() { - // TODO Auto-generated method stub return "login-time"; } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java index 92f5e7ba4f..1d64da9fdd 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java @@ -7,21 +7,8 @@ import org.adrianwalker.multilinestring.Multiline; public class JSParserTest extends ScriptParserTest { - - /** - { - "customerId":117, - "first_name":"karthik", - "last_name":"narayanan", - "age":"38", - "login-time":"2016-01-28 15:29:48", - "ip-address":"216.21.170.221", - "os":"windows 10", - "device":"Dell Inspiron" - } - */ - @Multiline - public String result=" {"+ + + public String result=" {"+ "\"source\":\"userlog\","+ "\"customerId\":117,"+ "\"first_name\":\"karthik\","+ @@ -44,38 +31,32 @@ public Map getTestData() { @Override public String getScriptPath() { - // TODO Auto-generated method stub return "../metron-integration-test/src/main/sample/scripts/test.js"; } @Override public String getParseFunction() { - // TODO Auto-generated method stub return "parse"; } @Override public String getLanguage() { - // TODO Auto-generated method stub return "js"; } @Override public List getTimeFields() { - // TODO Auto-generated method stub return null; } @Override public String getDateFormat() { - // TODO Auto-generated method stub return "yyyy-MM-dd HH:mm:ss"; } @Override public String getTimestampField() { - // TODO Auto-generated method stub - return "login_time"; + return "login-time"; } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java index 2bd1a82ffc..a6dc9b3769 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java @@ -7,20 +7,7 @@ import org.adrianwalker.multilinestring.Multiline; public class PythonParserTest extends ScriptParserTest { - - /** - { - "customerId":117, - "first_name":"karthik", - "last_name":"narayanan", - "age":"38", - "login-time":"2016-01-28 15:29:48", - "ip-address":"216.21.170.221", - "os":"windows 10", - "device":"Dell Inspiron" - } - */ - @Multiline + public String result=" {"+ "\"source\":\"userlog\","+ "\"customerId\":117,"+ @@ -44,38 +31,32 @@ public Map getTestData() { @Override public String getScriptPath() { - // TODO Auto-generated method stub return "../metron-integration-test/src/main/sample/scripts/test.py"; } @Override public String getParseFunction() { - // TODO Auto-generated method stub return "parse"; } @Override public String getLanguage() { - // TODO Auto-generated method stub return "python"; } @Override public List getTimeFields() { - // TODO Auto-generated method stub return null; } @Override public String getDateFormat() { - // TODO Auto-generated method stub return "yyyy-MM-dd HH:mm:ss"; } @Override public String getTimestampField() { - // TODO Auto-generated method stub - return "login_time"; + return "login-time"; } } From 0f15c73c8b40dbc756d91e70e48dcc4b7c1cbe75 Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Wed, 2 Nov 2016 22:18:12 -0400 Subject: [PATCH 5/7] fix for RAT check error --- .../src/main/sample/scripts/test.groovy | 17 +++++++++++++++++ .../src/main/sample/scripts/test.js | 17 +++++++++++++++++ .../src/main/sample/scripts/test.py | 16 ++++++++++++++++ .../apache/metron/parsers/ScriptParser.java | 18 ++++++++++++++++++ .../src/main/resources/scripts/groovy/common | 18 ++++++++++++++++++ .../src/main/resources/scripts/js/common | 18 ++++++++++++++++++ .../src/main/resources/scripts/python/common | 16 ++++++++++++++++ .../metron/parsers/GroovyParserTest.java | 18 ++++++++++++++++++ .../apache/metron/parsers/JSParserTest.java | 18 ++++++++++++++++++ .../metron/parsers/PythonParserTest.java | 18 ++++++++++++++++++ .../metron/parsers/ScriptParserTest.java | 18 ++++++++++++++++++ 11 files changed, 192 insertions(+) diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy b/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy index 44894fb92a..ac4d1ce7ba 100644 --- a/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.groovy @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ def parse(rawMessage){ String[] parts=rawMessage.split("\\|"); def message = MetronMessage(); diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.js b/metron-platform/metron-integration-test/src/main/sample/scripts/test.js index 3f409e4881..d6ce227de2 100644 --- a/metron-platform/metron-integration-test/src/main/sample/scripts/test.js +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ var parse =function(rawMessage){ parts=rawMessage.split("|"); var message = new MetronMessage("userlog"); diff --git a/metron-platform/metron-integration-test/src/main/sample/scripts/test.py b/metron-platform/metron-integration-test/src/main/sample/scripts/test.py index 80e37681a5..8f1d150352 100644 --- a/metron-platform/metron-integration-test/src/main/sample/scripts/test.py +++ b/metron-platform/metron-integration-test/src/main/sample/scripts/test.py @@ -1,3 +1,19 @@ + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + def parse(rawMessage): parts=rawMessage.split("|"); message = MetronMessage("userlog") diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java index 7d6e7a57a7..ef97cc85df 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.metron.parsers; import java.io.IOException; diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common b/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common index 132495d81e..849ee627d0 100644 --- a/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common +++ b/metron-platform/metron-parsers/src/main/resources/scripts/groovy/common @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + def MetronMessage(){ def message=[:]; message["source"]="userlog"; diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/js/common b/metron-platform/metron-parsers/src/main/resources/scripts/js/common index 2cf2075fc5..fea4b8494b 100644 --- a/metron-platform/metron-parsers/src/main/resources/scripts/js/common +++ b/metron-platform/metron-parsers/src/main/resources/scripts/js/common @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + function MetronMessage(name) { this.source=name } \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/scripts/python/common b/metron-platform/metron-parsers/src/main/resources/scripts/python/common index 03878271da..e175615ee8 100644 --- a/metron-platform/metron-parsers/src/main/resources/scripts/python/common +++ b/metron-platform/metron-parsers/src/main/resources/scripts/python/common @@ -1,3 +1,19 @@ + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + def MetronMessage(name): message={} message["source"]="userlog" diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java index 59183b3a56..d52d9afba1 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GroovyParserTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.metron.parsers; import java.util.HashMap; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java index 1d64da9fdd..7f8924c980 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/JSParserTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.metron.parsers; import java.util.HashMap; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java index a6dc9b3769..b7c16589fb 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/PythonParserTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.metron.parsers; import java.util.HashMap; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java index 8b54c75384..3dda06d2c3 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ScriptParserTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.metron.parsers; import java.io.IOException; From 5f2545ac0f42d0923a429b9cae766fb58aa6abfd Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Mon, 7 Nov 2016 13:12:38 -0500 Subject: [PATCH 6/7] added license for groovy and python jars. --- dependencies_with_url.csv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 67b5373840..c2a22c6119 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -218,3 +218,5 @@ org.yaml:snakeyaml:jar:1.15:compile,Apache License Version 2.0,http://www.snakey ring-cors:ring-cors:jar:0.1.5:compile,Eclipse Public License 1.0,https://github.com/r0man/ring-cors xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/ +org.codehause.groovy:groovy-all:jar:2.4.7:compile,ASLv2, +org.python:jython-standalone:jar:2.7.0:compile,JYTHON,http://www.jython.org/downloads.html \ No newline at end of file From 932934e634e8b2b46d96ed44d32ae5ed0b2858ad Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Mon, 7 Nov 2016 21:36:10 -0500 Subject: [PATCH 7/7] fixed a typo --- dependencies_with_url.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index c2a22c6119..dc8aea933a 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -218,5 +218,5 @@ org.yaml:snakeyaml:jar:1.15:compile,Apache License Version 2.0,http://www.snakey ring-cors:ring-cors:jar:0.1.5:compile,Eclipse Public License 1.0,https://github.com/r0man/ring-cors xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/ -org.codehause.groovy:groovy-all:jar:2.4.7:compile,ASLv2, +org.codehaus.groovy:groovy-all:jar:2.4.7:compile,ASLv2, org.python:jython-standalone:jar:2.7.0:compile,JYTHON,http://www.jython.org/downloads.html \ No newline at end of file