From b6d1f5a4703331336982a2da51a486709bb0d517 Mon Sep 17 00:00:00 2001 From: Kevin Jacobs Date: Tue, 20 Mar 2018 17:00:13 +0100 Subject: [PATCH 1/5] Chain link parsers. --- .../window/generated/WindowBaseListener.java | 2 +- .../client/window/generated/WindowLexer.java | 2 +- .../window/generated/WindowListener.java | 2 +- .../client/window/generated/WindowParser.java | 2 +- .../metron-parsers-contrib/.gitignore | 18 + .../metron-parsers-contrib/README.md | 351 ++++++++++++++++++ .../docs/img/chainlink.png | Bin 0 -> 13647 bytes .../docs/img/chainlinkio.png | Bin 0 -> 15438 bytes .../docs/img/chainparser.png | Bin 0 -> 12069 bytes .../metron-parsers-contrib/pom.xml | 143 +++++++ .../java/nl/qsight/chainlink/ChainLink.java | 79 ++++ .../java/nl/qsight/chainlink/ChainLinkIO.java | 76 ++++ .../nl/qsight/chainparser/ChainParser.java | 222 +++++++++++ .../main/java/nl/qsight/common/Constants.java | 28 ++ .../nl/qsight/links/fields/BlacklistLink.java | 83 +++++ .../nl/qsight/links/fields/IdentityLink.java | 39 ++ .../links/fields/NormalizeFieldLink.java | 56 +++ .../nl/qsight/links/fields/RenameLink.java | 88 +++++ .../nl/qsight/links/fields/RenderLink.java | 173 +++++++++ .../nl/qsight/links/fields/SelectLink.java | 74 ++++ .../nl/qsight/links/fields/TrimValueLink.java | 54 +++ .../nl/qsight/links/fields/WhitelistLink.java | 83 +++++ .../nl/qsight/links/io/JSONDecoderLink.java | 64 ++++ .../java/nl/qsight/links/io/KeyValueLink.java | 92 +++++ .../java/nl/qsight/links/io/RegexLink.java | 88 +++++ .../java/nl/qsight/links/io/SplitLink.java | 104 ++++++ .../nl/qsight/links/io/TimestampLink.java | 294 +++++++++++++++ .../java/nl/qsight/utils/ConfigUtils.java | 135 +++++++ .../main/java/nl/qsight/utils/JSONUtils.java | 54 +++ .../java/nl/qsight/utils/StringUtils.java | 98 +++++ .../nl/qsight/chainlink/TestChainLink.java | 57 +++ .../nl/qsight/chainlink/TestChainLinkIO.java | 151 ++++++++ .../qsight/chainparser/TestChainParser.java | 145 ++++++++ .../links/fields/TestBlacklistLink.java | 80 ++++ .../links/fields/TestNormalizeFieldLink.java | 66 ++++ .../qsight/links/fields/TestRenameLink.java | 76 ++++ .../qsight/links/fields/TestRenderLink.java | 113 ++++++ .../qsight/links/fields/TestSelectLink.java | 75 ++++ .../links/fields/TestTrimValueLink.java | 53 +++ .../links/fields/TestWhitelistLink.java | 81 ++++ .../qsight/links/io/TestJSONDecodeLink.java | 69 ++++ .../nl/qsight/links/io/TestKeyValueLink.java | 88 +++++ .../nl/qsight/links/io/TestRegexLink.java | 81 ++++ .../nl/qsight/links/io/TestSplitLink.java | 163 ++++++++ .../nl/qsight/links/io/TestTimestampLink.java | 153 ++++++++ .../java/nl/qsight/parserconfig/TestDhcp.java | 26 ++ .../nl/qsight/parserconfig/TestParser.java | 192 ++++++++++ .../nl/qsight/parserconfig/TestSuricata.java | 26 ++ .../java/nl/qsight/parserconfig/TestYaf.java | 26 ++ .../java/nl/qsight/utils/TestConfigUtils.java | 70 ++++ .../java/nl/qsight/utils/TestJSONUtils.java | 72 ++++ .../java/nl/qsight/utils/TestStringUtils.java | 42 +++ .../src/test/resources/dhcp/config.json | 72 ++++ .../src/test/resources/dhcp/data_input | 1 + .../src/test/resources/dhcp/data_output | 1 + .../src/test/resources/suricata/config.json | 42 +++ .../src/test/resources/suricata/data_input | 3 + .../src/test/resources/suricata/data_output | 3 + .../src/test/resources/yaf/config.json | 69 ++++ .../src/test/resources/yaf/data_input | 1 + .../src/test/resources/yaf/data_output | 1 + metron-platform/pom.xml | 1 + .../common/generated/StellarBaseListener.java | 2 +- .../common/generated/StellarLexer.java | 2 +- .../common/generated/StellarListener.java | 2 +- .../common/generated/StellarParser.java | 2 +- 66 files changed, 4603 insertions(+), 8 deletions(-) create mode 100644 metron-platform/metron-parsers-contrib/.gitignore create mode 100644 metron-platform/metron-parsers-contrib/README.md create mode 100644 metron-platform/metron-parsers-contrib/docs/img/chainlink.png create mode 100644 metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png create mode 100644 metron-platform/metron-parsers-contrib/docs/img/chainparser.png create mode 100644 metron-platform/metron-parsers-contrib/pom.xml create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java index 163cf1f9be..3296cc6e28 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java index c10764b1bc..20c54124da 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java index b831f31119..1c3ed5e7db 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java index 415d540fd4..660114cffb 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-platform/metron-parsers-contrib/.gitignore b/metron-platform/metron-parsers-contrib/.gitignore new file mode 100644 index 0000000000..1efb3f3ac9 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/.gitignore @@ -0,0 +1,18 @@ +# Compiled class file +*.class + +# Log file +*.log + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# Editor output +target/ +.idea/ +*.iml \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/README.md b/metron-platform/metron-parsers-contrib/README.md new file mode 100644 index 0000000000..35e446a6dd --- /dev/null +++ b/metron-platform/metron-parsers-contrib/README.md @@ -0,0 +1,351 @@ +# Parsers for Metron + +This repository contains parsers for Metron. + +## Parser components + +### ChainParser + +![ChainParser](docs/img/chainparser.png) + +The ChainParser first converts a raw text (the bytes of a raw text) to an initial JSON object with two fields: +- `original_string`: The original string object (which is equal to the raw text). +- `timestamp`: The current timestamp. + +These fields can be overwritten by ChainLinks. The ChainParser validates the intermediate JSON objects and checks whether the `original_string` field and `timestamp` field are available after a ChainLink is executed. + +The ChainParser points to a single ChainLink which is the root of the ChainLink DAG (Directed Acyclic Graph). A ChainLink points either to another ChainLink or is the final ChainLink. The resulting JSON object is the output of the ChainParser. + +### ChainLink + +![ChainLink](docs/img/chainlink.png) + +A ChainLink has a JSON Object as input and produces (potentially a modified) JSON object as output. + +#### Identity Link + +The identity link passes on the given JSON object. No changes are made to the intermediate JSON object. + +```json +{ + "chain": ["identity"], + "parsers": { + "identity": { + "class": "nl.qsight.links.fields.IdentityLink" + } + } +} +``` + +#### Normalize Field Link + +The normalize field link applies the normalize function to all fields. The field "Hello World" is converted to "hello_world" for example. The transformations are found in the normalize function. + +```json +{ + "chain": ["normalize_fields"], + "parsers": { + "normalize_fields": { + "class": "nl.qsight.links.fields.NormalizeFieldLink" + } + } +} +``` + +#### Rename Link + +The rename link renames field names. The following configuration renames "field1" to "fieldX" and renames "field2" to "fieldY". The renames are specified by the "renames" argument where the keys are the original field names and the values are the new field names. + +```json +{ + "chain": ["rename_fields"], + "parsers": { + "rename_fields": { + "class": "nl.qsight.links.fields.RenameLink", + "renames": { + "field1": "fieldX", + "field2": "fieldY" + } + } + } +} +``` + +#### Render link + +The render link converts a template to a field and substitute variables in the template. + +The template is a string and might contain references to variables by specifying the variable names between "{{" and "}}". The variables are substituted by the corresponding fields of the intermediate JSON object. The "variables" argument specifies which variables should be substituted in the template. The "output" specifies the field in which the rendered template should be stored. + +The following example substitutes variables "var1" and "var2" in the given template: + +```json +{ + "chain": ["render"], + "parsers": { + "render": { + "class": "nl.qsight.links.fields.RenderLink", + "template": "Hello {{var1}} and {{var2}}", + "variables": ["var1", "var2"], + "output": "rendered_field" + } + } +} +``` + +#### Select link + +The select link selects a fields and stores it into a special input field which is used by all ChainLinkIO parsers. + +The following example selects "field1" and stores its value into the special input field. The field which is selected is specified by the "field" argument. + +```json +{ + "chain": ["select"], + "parsers": { + "select": { + "class": "nl.qsight.links.fields.SelectLink", + "field": "field1" + } + } +} +``` + +#### Trim value link + +The trim value link trims whitespace for each value in the intermediate JSON object. So " hello world " is transformed into "hello world". + +```json +{ + "chain": ["trim_values"], + "parsers": { + "trim_values": { + "class": "nl.qsight.links.fields.TrimValueLink" + } + } +} +``` + +#### Blacklist Link + +The blacklist link removes specified fields from the input. + +The following example removes "field1" and "field2" from the input. The fields are specified by the "fields" argument. + +```json +{ + "chain": ["blacklist"], + "parsers": { + "blacklist": { + "class": "nl.qsight.links.fields.BlacklistLink", + "fields": ["field1", "field2"] + } + } +} +``` + +#### Whitelist link + +The whitelist link only allowed whitelisted fields and removes all other fields. Required fields (`original_string` and `timestamp`) are automatically whitelisted. + +The following example whitelists "field1" and "field2" so these are the only fields left in the output. The fields are specified by the "fields" argument. + +```json +{ + "chain": ["whitelist"], + "parsers": { + "trim_values": { + "class": "nl.qsight.links.fields.WhitelistLink", + "fields": ["field1", "field2"] + } + } +} +``` + +### ChainLinkIO + +![ChainLinkIO](docs/img/chainlinkio.png) + +ChainLinkIO is a specialized version of a ChainLink which uses one input field as input and produces a JSON object as output. This JSON object is then merged with the intermediate JSON object. Before a ChainLinkIO can be executed, a field must be selected and stored in the special input field. Instead of inserting a SelectLink before every ChainLinkIO, it is also possible to specify an "input" argument. The "input" argument is automatically transformed into a RenderLink and stores the result into the special input field. Note that it is possible to parse variables in the input, for example with `{"input": "{{var1}}"}`. It is also possible to use constants as input: `{"input": "constant string"}`. If no input field is specified, `original_string` is used by default. + +#### JSON Decoder link + +The JSON Decoder link decodes JSON found in the input field. + +```json +{ + "chain": ["json_decoder"], + "parsers": { + "json_decoder": { + "class": "nl.qsight.links.io.JSONDecoderLink" + } + } +} +``` + +#### Key-Value link + +The key-value link spits on pair-delimiters and on key-value delimiters. For example, take a look at the following string: + +`KEY1=value1|KEY2=value2|KEY3=value3` + +The pair delimiter would be `|` and the key-value delimiter is `=`. The following configuration is used for parsing the example: + +```json +{ + "chain": ["keyvalue"], + "parsers": { + "keyvalue": { + "class": "nl.qsight.links.io.KeyValueLink", + "pair_delimiter": "|", + "key_value_delimiter": "=", + "valid_key_characters": "A-Z" + } + } +} +``` + +The "pair_delimiter" argument specifies which pair delimiter is used. The "key_value_delimiter" argument specifies the key-value delimiter and the "valid_key_characters" argument specifies the characters of which a key exists. The last argument is a substring of the regular expression for detecting keys and is required for performance issues. The value "A-Z" refers to the fact that the keys consist only of uppercase characters. + +#### Regex link + +The regex link executes a regular expression on the input field and uses the first found result for the creation of the output object. + +Take a look at the following example: + +```json +{ + "chain": ["regex"], + "parsers": { + "regex": { + "class": "nl.qsight.links.io.RegexLink", + "pattern": "(?i)(user|username)[=:](\\w+)", + "selector": { + "username": "2" + } + } + } +} +``` + +This RegexLink searches for the pattern specified by the "pattern" field. The selector specifies the output fields. Here, the output field "username" contains the result of 2nd group of the regular expression. The "selector" argument is a mapping from output field names to the desired regular expression group which is used as value. + +#### Split link + +The split link splits the input on a given delimiter. Consider the following input: + +`value1|value2|value3` + +The split link can split on the `|` delimiter to obtain all the fields. + +Take a look at the following example: + +```json +{ + "chain": ["split"], + "parsers": { + "split": { + "class": "nl.qsight.links.io.SplitLink", + "delimiter": "|", + "selector": { + "-1": "last_field", + "0": "first_field", + "1": "second_field" + } + } + } +} +``` + +When applied on the given example, "last_field" contains "value3", "first_field" contains "value1" and "second_field" contains "value2". The "delimiter" argument specifies the delimiter to split on and the "selector" argument is a mapping from indices to fields. Negative indices might be used to traverse the items in negative order. + +#### Timestamp link + +The timestamp link uses regular expressions to search for datetime patterns in the input. Take a look at the following example. + +```json +{ + "chain": ["parse_datetime"], + "parsers": { + "parse_datetime": { + "class": "nl.qsight.links.io.TimestampLink", + "patterns": [ + ["([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", "yyyy MM dd HH mm ss SSSSSS Z", "newest"] + ] + } + } +} +``` + +There is one argument "patterns" which specifies all the patterns. A pattern is a tuple consisting of multiple components: + +- Regular expression. +- Matcher string. +- Ordering (optional). + +Multiple patterns might be specified. The first matching pattern is used as output. + +The regular expression is used for detecting a given datetime pattern in the input. Then, all the found groups are stored space-separated in an intermediate variable. The matcher string specifies the space-separated datetime components of the result. This is required so the timestamp parser can convert the datetime string to other formats. The ordering specifies which result must be used. The ordering `newest` is used for using the latest datetime string. + +The timestamp parser generates the following fields: + +- `datetime`: The formatted version of the datetime pattern found in the input. The format is specified in the `mapping` field. +- `mapping`: The mapping used for constructing the `datetime` field. +- `timezone_available`: Whether timezone information was found in the input. +- `original_timestamp`: The exact match which was found in the input. + +## Development + +### Creating a new ChainParser + +A ChainParser mainly consists of configuration and uses the ChainParser base class. The configuration should be created under `test/resources/your_parser_name/config.json`. Take a look at different ChainParser configuration files for inspiration. Then, a test file need to be created (`test/java/nl/qsight/parserconfig/TestYourParserName`). It should contain the following contents: + +```java +package nl.qsight.parserconfig; + +public class TestYourParserName extends TestParser { + + @Override + public String getFolder() { + return "your_parser_name"; + } +} +``` + +The only thing that need to be changed, is the `getFolder()` method, which should point to the name of the folder which was created. + +The test will run over specified log lines. These log lines must be stored in `test/resources/your_parser_name/data_input` in which every line represents a log line. It will also need expected outputs. These are stored in `test/resources/your_parser_name/data_output`. Initially, you can put empty JSON objects for every input log line. + +Then, you can run the test. It will show the duration for each of the links it consists of: + +```text +=================================================================================== +Start ChainParser: +Epoch: 0 +Logline: 0 +Input: ... +Expected output: {...} +=================================================================================== +Link: nl.qsight.links.io.JSONDecoderLink +Duration: 12.724098 ms +----------------------------------------------------------------------------------- +Link: nl.qsight.links.fields.RenderLink +Duration: 0.043077 ms +----------------------------------------------------------------------------------- +Link: nl.qsight.links.io.RegexLink +Duration: 1.275897 ms +----------------------------------------------------------------------------------- +Link: nl.qsight.links.fields.RenameLink +Duration: 0.041026 ms +----------------------------------------------------------------------------------- +Link: nl.qsight.links.fields.RenderLink +Duration: 0.03159 ms +----------------------------------------------------------------------------------- +Link: nl.qsight.links.io.TimestampLink +Duration: 2.045538 ms +----------------------------------------------------------------------------------- +Parser duration: 16.161226 ms +----------------------------------------------------------------------------------- +``` + +This is useful debug information and shows the bottlenecks of your parser. \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainlink.png b/metron-platform/metron-parsers-contrib/docs/img/chainlink.png new file mode 100644 index 0000000000000000000000000000000000000000..de4e71190a308b84530ea488c658e67f9b84275e GIT binary patch literal 13647 zcmeIZcT`i`8ZSx*L6IU=PFqA`+_U#S=b!h+xNnSi?|T_QSjk#hbAEHqZ~n@cNFA{14Pts?EG(=W>S`c8 zEG%3$;C~De4)9lGJ?#v5!uHTpRm7?qV%Pu<@az?|6tJ+~#gm-b-~-3ko~W66U|~^s zUj1T!hTiqT!V+Ck2Pqi%SnlL7$1tLNk>c2gX4r|TK`}Te}rvqZB18Oc^GXz2)sD^^5x5TiT3Ibn$N9%>C~Uw z1FfRkIvP{bVcAyo7mqI{e zXt{k(esOc)E(XOUDC%kzk%$Q(n`Wdv;Xps7pt+wm=R&2y^=3~kOhwjNq zLk%3ryA=KWTmFsSpwRXOS8W74rsMhHE+b1&!?zby#i5ve##!%z`{747Yy$do&>Crs zj0#>%l3+DV|MA>h?=N z)D4CXqLt9PionKS< zu_IWXf}a)bu{6D10DC>D%jEYA-OclRYS<~#Su~%gCF3Y`Bx{X`MeO2#L65Xj$-ppUY%|A}60bCH%vS9juf(VigoAzxG%L+?@5% z>pFcU5wgxtW{27t>QZtVlG^H{*4p+SYBm-LazVFy&Q_YY~!U+I-M5 zFx>7GcT!{yxGCB$~%oH$CO4~W2QN6V0qBUT9t8Bh52g+ z(AV`fQ12(p?Eo>zryyWeN;C1VGcg@-(n4`ut`)H&-Oi=Lq7+ zMYZQ(F!0%q!O_{t=}M^gw?rjwn#6FGU*)IizNszSZm~q@Aw5q_63jkQ76Ru+OzvKL z=KMSfjGA8dElWt(QxX~32n^tohy8WanU!Q@7JPhs{J_5J#q%I%?&FXQxOT-QEeIZD z$P!J)Hs}8IX=nby$J%}8!1EI^c3{9n)e2IgdU0`W6h`ve;BF}C{e#?%yazcCy?)>hk`|5Sqq}Fsq(F>rj!YwcgJd3 z;XE>d1XF|5xN$3bRp=(j)hv_F;>Wp0#k3irJ=Uvmih5zmO6jPEl}CCUJ}4Fw`Pqlw zEKhMgR^t44Z{1+%cI}Ca?6sjjW_c0^?Abuoj}jseD9T<@Xh%I<<@=C8B z(zjsd+St-EaP&@fqEa}bY0Rv(j*D|=f_iPZqY3>ohCvkaEcz;oLNcbpx-x+oBT>?X z)-czl<-}v#)T|^<%w?nvBq#rp0Nto!uvEvYRmAUW4)l6>f89NTsk$?sU~=&Y>#aOf z6A|G=xPrUrq?$f_vbrWF{gr6!;Twrj<9xGdSSIvM+TkOG&9wMf z6s$3WLF$^U*c*zw35#BDXY*I7e+iX%q!epxXh6w(%9D>c6EG)H@_aN-7?zAt=7LBj=VsRi@Gz4jHokl@5K`Uzx$Q9}zj{a7S zO-)X6trX;mj6Qp7TIWL+EAl{s10JLiz(VQk%Ln@a$7y z$|(;^L;vOZ;^q0c+`0ZW<;L~1{`ND?#_C$S!6~fvQ()shom)(cnYRx(S`Po-c3K5& zvEIi&KizFVKbT8O^cxaibYN?X^AHV!I^P3&x=S`Vl{5ky?3ARJN7T~-fqNDKbyjyk zt}QX#$U~c52a=vN2vN@_3h{UnM=%28Cm2lYWh_9OQDnj;B~)=8kklbi%@xv782`1w zDTeGqY*9$G0)pK=+yR*u3UwwO8pXbT}y8Z*R+P`zLbbfkhV4c7oSke-3p4ll> zlq4s7)#YBTfvFPh+@ay&+y~DwM?b$BS6jaQ@iF|S`T%y~S-F%hHi|z(n%#E)U}r8# zPWDeM-bbPp>F>z+f_@c{{!}U{DXAPi_73%#Yb%aa=P}qXTk_LkZ{j_M1V`EmT`yZX zR4q#D86Fza7?rq+=?Vvwgo^!4m$pv8{qRdSiN@!a=gGuYVTiVJuz^WB(E(~ZP4d^z zub=cheS9oOz6N`bKBU(s|Lxq%IO_K)dG}+;pdp1slADAfk8hNtNU@<`Zx; z;LpB)mU)f|O=B&2>&fZrbAk)Z32S=<)oss@>0^b^XcLy`+~`S25Zd-)&-(`uNzp1` zP)mS_F9;pj5F)}b5ByI=n>zDPU6CyLPfO>C##TGP^FMwOzVs07aD{Gt`NA?3!o}W; zD$bsoZO~aB%n}#0Y*Evi*WS18F-dC>wW8&jr|MDp%+GMNJAWDUhtqnr-_Y}V!XE;*L0 z)qh>ybIaXf0d_B`s#9a4x(}Ks8@7KdF6r%4rqMokd!)pKAr`?+_PwxPu^X76Zvl^7 za!E7cb63FfPZMgV?Di4YErv~meZcv>xUs+_54Dq_(TYRmQ)E9(T&|gsO;fROP2PSgX%{|V zG?7o0Sd|E5=oMK-5NF3T5r~bf;ns{A;0WU zDT_3#sw{y>QwRR%V!_&K79-qKFLk>IiIkz12}|9s+7!1;@!W4V$=PNv`k9oWRnT;Q z#nc*rQFkL9l>I1e}cd7+iN9s*cjRsuDPibegz%+?D%mtTw za+e;2%EVjm7Be~VqCC%Y-YMGn^Da(>ABgEaZcR@pRv9!pRKiBa!%mZN4oF)cTTkSZ zyaIyYVQg_*M?glkTO?Or@N*5@uhM)L_nmZW*nIU=Z)@+iI^6;k<0_J$s9JxY_?lbf-g$#!c$_}2+< z?!Kj>r_TgT0Sp>0tZ?~yOcV;|;NVk&e;~P$jM z-hOi-5A13Z+2}sMoL&2_IAp+l4|@3Wqs3LL=XA2Qy{+r!UAjl2VT4=JRYMPe6=_rn z&9AsADsi0VmyHGX&37H#l`o_i};Ig@qK?9 zjl{g13@YN>*!-%=w9&^A7D@Wu8i_I7>h4yi*ymK1Kh$aWW^KTL$DvyQt>;Tm0A|QX z#N_meVRpmENCSb4ocGHcH}TdTf<9*OFDZaqTq2jN0xWM$pUjAvnS2LU+bi3!pzq#l zjIc%t8am||vyDh;m}UV{)_{?$0=(Uig|{)=TXru;VZq;Zc?}vJyz@k=a1s_-DRPV6 zVWR!=v|XELjA80gyNEVZz9HS!t#gG@v$7|n9rCCo*wDkxz+;)Puom)j?g0ZuRe7I1iP!Rli@iSaX2oTM$O=S2KoWh-W0kK3=F30} zy(Rx zG+v@>U^J8F@qv0TNkHk`5Lp9=gpP+*;c)Y8ghO>MjWy`(En~UO(h+0}*EdTl=EvTC zwd`IVeWXFhEtc-wYWsbDW8d)oz*td2upl=&#zPdVKPQ1VgM6l}Cazh+ZeQO-n0JK8 z_EG(soRUyoN#(REpWa>l)XbtS?q2k|(Uil-$Z^T^0L=j6TTzJ8Dc9~GL^)QXe9aLg zhXKOdH#ZEY)xPn)1zdRO`aKLUPppj6q2%Cpc-_!Jx<@jA=MH)YMtVbXyR-(V6X!Ch znu=@At2vqIi5Bd=j&L;etfb-7_74o77f71x&S@^ZN9vf6R$sik&psz!Wta@H!E1VC z=1yc-MBXFMqRMKtvuW-9LccL9ek=NJIsgP&HnIt50Sfz>BMGeNbR9JVs%W4{aSH zU;qsgrpe`ENv5S|0oG*XKsRSpFWakEz1~-6{fuN0j?8DSWdsnS`HO;x|I;st{Tad+ zcBM_uPoU?kIf-dZ1Ppg2zkhD!72h{OGmBoyqypNzl6PfaaETkDHG!wa zP1iVFb6bg31A70kALf6$(;(f_n#>wn-xtN7QD`SBYLZbo{NL1U5~pKU>bwSzq#2*%)r#A_`6nskSEOhhnfBD6KZGs8KB>R4a#~A@9*yiB81>t{_^EzQbh$X z#hwyT=R#ZYH8q}^ZoXqGwH|w|reM_E4UVYJ5jWKLn*wacU2jtho=W(aK`W}pr0+p0 z*XJ3zlzDq=@3__)FDELscn-RT_hnI|9Gu1PUTTfN>V>EbHTM?N3hmSYXbLF^Q8M`E zdKiB5?l-QK0tmLcy^!CwsPyuFe|*K9f+($}ItmKFZ14LJzAwpCYI2&A7017RrHDx_ z-iyiPuiYrX{Ms02#Bmg7vG==0)KMcG-kA1d8!fViWVjO?%s&RJ*Zh?>NKHJn!zB*w z29kqi+lvfUYwC5^Wy+cC5gdz@v+Jep8M8OUncM(yRczRh{gWYUytyHSmU)W_Y?G!DL&WzeU)-U!9d8LP`ul)m%7KzChs*1oo1 zt289g5snT@ZOlS{>3PbkgGjRK^tTIGbLKuwR9Va7H*&8_i3?J0d+fNA)0)Nq{{4Fg zK{D@|0RxG5MRpuv&fAW_h|894divJoubZrNQ zl)7DaTepON^yj3dN?`i?YvDqW`M8z}O@y5`7(6nzYcc)Mq2y09nlt!C!|F}S)|R%n zWBdchf@SoH%X&;-BaHF^4bd+V@48FvVc(QpGWw-~>)=$DxZ8dItqy8?nSF|a7z1llN@jN+xYAf7U(ZPS)nuWg2#m-QD#>=U7Ezq*qO@o2qyzviH$h@Qbm0}l0g z%rfsEYc}QF?G+Q2481c)=ev# z1GcM}J($;j^J3oCm4SV7w`NT;>YF|#=4j=4MqvW==1cW7s`q6IO#N&r1H*JKaQN_b z`+9wT)tfgOQn6D8TEjNVDTn&zJ)MyqEX#31djoP~2I5honl=_Yc#3s#CtMV&C}dV} zRe(21OgO=)jFVn&lGH4z)MGe{5nS}CRLR57#jp@lWA=8IXnul-xNj*{k~y8AO;j<| z9@zxTE20w3eC%^QNyQSKF)3*fpucst_wa%{-M3>jS<3Bc*KQ3~58BJdV;jc?hsgjc{KdVlE+E=(8ho_e+hctit45S}RGFMIA(p)@^?WUL zV0Z^S=<1e0yo2Kk#APaeyr5;5*A;u08<#8N^kJ%w(ONMCk>j<1ReNh{GA@4E2&q9~ ztefs&oO@g2;A8sDqxbu!V-|v`+X79)ie#3zDtGFV>bnxuibC5Q@)tDgMu=2L8|-Y*Yocnty)YL?ek!iz&H?>o*ycRx!*2dVABlp3$^k%= zGm_mHFnNohck#bY=$OJ8{=rT#*z3;Q@*zjkq&rVSOv2A2H$kJzk&@{i(T)6Lp zNO3Bg2JP*_f@~*%K&`|u*P}}z-&Xk0>QjS8p&|BX37_kQYK*d|RnDfY&bqDYC<7#F zc}s65sn<$u?T{{%Tj$FLUbLLUNZg5j_v?fMTH(S^g#kz3S5?}0^+cRb9yK2rvAB4) zCQc{};R_f`o@h0N$lac<>O@%v^(0EGyzy91CfMpPf1O|S5y)Q>^jnnW^0ArlnL)t7 zOg;C=zU$na68CZAkDv+A@F@7kfc_`n^P#YZ2epWC;^$3Ejy@@4KBv)^pY4U9Cyx&o zQTi*DJc_kDYh@Kyi#zPi_&xH2SrYt5xtGVea>vPXHX7-D)q&?*a%Ugq82s6`d$Jp1 zOtUSH>XYG*I3(mz;T ztXf@Q(%bpc#T83#=eO!7uay-7f=0y+;^fx6re;YvStsaQtznzVc?n$U;mtr(sAD!C zBBc-zN>kO5e#Q;pG*32hT!(N`y!EscSdS$A(i14JU!uU0^z6wy4hxA@f9*LEcJZfX z3F}-tLdf@3DXKAMZWF(|_u<5OZ$Mo-m&o++{6dO6Q)@9}F#Pl7?)QthcFnNU?iafU z2DD+5y|Y164~^YRX!V{oGe@(93}9SooSK*NotGkZ+|u34pLZic zJvG;7HqSPV+fNX3rwI4wte#Sov(orQ4ZP~cxo`d4-IF}66D$PL^}E}3OpET;4knVK z$`eJ|Sj=;gl42FlH|Gt#vv%6VJ@mG4zp|Y*M6iXI>W3@L-@~=r3U^i-@%sZ?NbQkK zxprg9sA{GU&K;in2<`IyuNAYFT+EvF{LHE)2BkLb;5UUcVwp}>Mcs@>VFl4K?Z6bR zBnF*3@P#Zljl4w4g7lkAZN*K))5GUo8}dSe>u2kPUbF08Xx+J9*p=d;Ehs4X==D+m zUeGpIPq#=?%rcI_LBkZLjkOZfY@p=ytAe_DKBF-x+OU_tmSl{kUox)w^CoTlEA z-|Y>Z&8t2uUqI74cgiJuE~}gDbhfI{eKdCrq8X!^#vGrtzqpZh$Ubm$Lmhb;Bz6tA z@I1+s`XM)OFy45x^2s4-&RRhIlkqLqU>J?8DH|PIMYN$pS_u(rNg0e&kFtrnZI-F1 zGmh`;Rd6aNx8;36Z&&hajILI+y~%ZR%{M3Wt9nv6oUZNq9Sf1O2J9$}(m7<6Y~;Mw zX5m|dp}E3Y&A2M1=RESiyiZ;Fyva9GBb z*ryY2`K3ve;7WLx;^~ewBIAkCzY#8;pkp$Zcmu5!Y2@1e6H$h}mmJ3%qCuiuFx=NncymsR~q z6hVdThxJiyU1bm1>`BLQVQYV3NA<;8G>$7ZWk1V6tFxayyH4*Y02>!d+$~nJEr;0$ z61=FGkw1` zW>A-6Q+*mhs@vwYZ*SJu%WXpqwf6MaO?107zC7);Ca4bSUs4;brFUNHVHyI z+GH`@h#td2tDj&KL3ggeqxvV6O)qZI(ZacMXGcSgJ-kKQ7XdH*SS6vMqXv>ENE zOQ^G`=sz?f;4YXf`078&ut_-@W5p^!@z0yG0Iq-C9~<-Uv=soqIpaA0-K$}kg_Tv{ z(T{vPqdzX@-uxTE(oq`M4l>c%@TNWb~g4Ls%MK{eMWz`M;)RiFL%wT^s==bKmXAw|~HX0E!pN z!NJ_h%0uJIKyP_*{FhjR*bkWxv}_NERY^IOx(#SzY=!xqJ^L#wMWDiYkG5kK)}LMg zG1s9KeQUQO8*)`TaL`Q(-I{Z2=cj92|4YmOOh=glv~TbKzOOt{LU=uYe(cl@s1zEb zM8Ad_*+semp``4`?m}}?yZ#lCw5HWV-KD$gt9w7R!fC_R{h60ncO&5A@*Im*Rach< zX{4OxP}s;Nbv`WkolZIcVVbj+<;rINwV}@voXZ)80B5uQPWSZZuzTTD^tH^j0k8Q=cO2<=#;xO37w*tm?h9V~^RG zUSYqYjE)5)@~Y>+Y81Z*0m&=}`ukg8#1f+2-@JVCC7%gqgPD=GS(0@cO2f zmK6v4BU05BSUa?0E}&bh0KqQz5)gmmV_x+PL3wXaFL=JExVqoc{?>HPgx%Nh(zG*X z$xyWI%FcOIeo^{D3+umN$(2Vvu2JbOV57g-_2TG<%{lzKbB>fteP0o}AaTMz_tWCe zJ(MoM58DXlK=gbWf_a{|kl4qK->DXm!tN^1Yw7Hp>F_nC7SGld6}e}z*2lUxHXKp$ z@?U{q_}`Xr<_-jXkY#tibzBoD^5kDoHlO}9{I zwhwCd??k_ox8V&k9LP`AsP!}slNpY1{gLc`gFAfxv$T7#1wv%G>koh3h9d9C1q_(M z)SuhGiKvQ>j)n>1+%HIt+&@W-c>=&!ZMypz{9j@VK?bO5c|iwnVNyxZIk@DijFDuq z<+)Oc9j~aN7|Wx=@%wWsn5JxO7v#9|=~k|W!F$FDn`l!EtB??|=-X&--1rbkh)9hm zG-Ki3!i+g`Yv|PzO>MQmI2IPezSalXGs=0QD5@sy!+4=rl05k0QdIKVy@&$gt{fw!(;lP*f8vlg2dZ@so zswm34D(ba~h;kYfa#N+rIoINmhC8Y*#zg3Ymte2q2zmI(7IN_TuTFV4M@KbUAE z1PWx(2@aLH_;vf}K%Lwt8uvl0vzB9uc^F?l^352y-4v4%B)IniCS$I|`7-g%tWT}s zLSDs9&reJcnW*JQ>82Pf4rWici&xgx|C(c=m&*gx!ejmx3F3x^%~M;?z@@HdKmi36 zt;njy*Wf9g)cqWri0LwYkp62VztsrRX;DV@%5cY3MHwC6Sf~@C-NYmKc&lDS2#-JI zHx>z+M29UtY@@%kEvF`Dac<1ab4-9Ri$%)%F^w%)Px+~z;l{3{V$f*xSwsOhxC|$l z29zWvrcQHIb5hK|nkYhAk1ltWZ-Nh(_7>n*XbO6|;84~#<=vIgcpRr!!f-p78pv6v zwxXg$mw*g_7bPmoK~GV;Q(2uedjd08e|CETWm!81!aa@-N*Q|&TCP}o=xRRNk&5)E zZg}66qOX|w0R{UuVDyF&sFTbUydZJO>HxugPp`^o_V4<-2}jR4W~FKYG7kf=)^IxI zZ3x5l!aa@KouAa!u#?wsOioB8@}Iyr4sf}zTUp8?cniP-<~g~&ri?EtT=SC=c4$BU zwzXWuMUQNr7Q<>K|Du=6%iyQ3sDW3bPDs!c-HjVx&gG%Ji;7x2>gTC|go0l3wUM2~ znILHIUQ%7ey%#j=c)w~NmpikddPX8NCG+^5u&T68?!>XErLa})+0His1b)V9iqZ1= zxHo^+zl`@b`%}$KWWZ;bSMWWwX8%g_&z%*K=daQaf5739zb3w4mwL{kkOrt12gdDH zSErr!S0`D4*XKU<9Rk*SwAI;557jbbHWn(JMC7x5cg`|l(qZ%z53xEp)U^w}=f zMjm4BM#>XYv^H* z!P?pl+i6oZ|1#DaU(yNh{la`pD4u;na(mVC_8ZteWDzEe1o-U6i zb8+i_AV<%(AR{Y`yVI6pGXjI(+^|*K9M=RqR>$VFe~qZZ03kET8i8U8sNr~@aZ=t` zk^4P^tLQ0sxvLAZgMVy&7y>E4vD*t(nIK>mtdN6UESiIuFagbsa;w>zk~_)J(v8+D z9Vj3`?cKv) z*y2*u+H?FrhrtOMbYQxB%ZTWde!#-NkFp@Rmsv-B!g6N53(K(7YL%m?c?X+1{gu4u zI?Ka)|5=+KKn;^u0E>*dPpv#Yy}G~C79<6(C|`nqD*{+GWtY$g8>ot&41T=Cx;R@ zx`psWt3)!hdSn)Rgp{JG{M3}-UD8G?gJm!b8cM~HohhkNfFvu8;qM>Sm1zB=I|6P4 zGZ95)$C;P*s#EWjf>>$IT)<<}7?0AYo6m|}a<)I-KeuOe2rh){>|*g>0ZRN2KGIO< zh#ieuQn6=qEjwXcfvABX$?Uz?O9c+zn5MC>XUFJ&l~y)90_s|UQp)Ybuc8ol>3_ma zj3Htl0Ba3{a(4e&o>{Bf=7ai}zW5K?2-I%=elzci@%^(p^Prajfj{Zy_#cIx zXjYLy>MpnQ!@$di-{#=oYDZ;W3cbnAr8M$5x;Xs~sB%`98@U&blBTvRJ#TH#IB?&%{is{859L1m7vBX^=QDi8ldQ$Fo?LR8kiS`=6pfSx1p!R$|1415 z(Pszj%YVMcKfYP=pQWvHp%KpA`u}<6|N8P)Ea(>Sy_o8iix9^tpzf9GH5@?F=?*D= zaeul3@HrY9n#cYqnf36y0bGZMAXmnND)w8eZ#;mn9=VpozLex%$Qg1nlJB}PL;_!e O!ctcTgQ^s*U;Yn)VuDBj literal 0 HcmV?d00001 diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png b/metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png new file mode 100644 index 0000000000000000000000000000000000000000..cb1ab354bc143dcc3a4bc9f46e62af96c54b19e9 GIT binary patch literal 15438 zcmc(`cT|&GyDzG;6qF_+oe=3tFVfpmx_~IX3ZY79A{am*ECdN10R^PUQi@U~BE1`` zfGnj2kQzXd8hQ!InL+)&eeSq>jC1!s_nbc*3{&1&-uXVyuO<4{O>Nq9?B`CLI6Sw6C8i`^32j{_}_HHG^v>PM{O139!@P|7Y*% zSo)qgalw!L@8qNlG~mPuC7iCtHIsllD-&vWZN`x`AuJ!`sRW`fc?m@O-h1glMHe+~ zbMd?y^cFV*6Z76*v~~93)YK-}w#TQ1^o5wJZ$0qwp-vMrjksh~Yf@;`Fm%N<=Hj*a zKxn^z(8Rn-Zql%au5?BJ=9OMzg$s#ISGs?+BHwq7MK_r*J{%&Tn|zfEk)Rc z=F`Q&P~a5#9=`9x>xq25cbuC8&lH8pP*WOSD$bibG=s;Kab zt0D<|`{B8CyI-j%hroSYoMk~sRpxBl!fCF*sXUM zKHY%`Bko!%>WxRk(flYB>OD2JKLX)^m1%sq{UeoqUeQ`$kQLKvzNwNm*sn&i>9e7* zF2@w4fo;eTz+byV67>J}?OXOuR|Fzeq?p+j^Q=BR6qRRTV^gxw%+iXku4bt?3QK0`#^WUi6xhWgCHuN^b923_ zIf}t#EtXiB?^NNrt#4BO4WE#6cXvm}ERLlr@K&A2r0WQ>!g1naVv~(QX&v64o-51C z5=l@sCcTP|6I#P!U5jF)V227B+dfGfE_?4xl+j zUbawOJQw-6#F>46D3_Z9`fi@DqfMrQHAVK$xD)-@nbRt~yh1Lixt^DwVStR3kQl_H!0MB>vPAZ)@(G32Ys_O1oH?Y{| z7Zrrv`)MjBeBFJ6S^Kn45HxF7B&w{2SX zegCvSsFz%z`*rf-gX`8=IXRo&1ZT>*>KtuWaO9;!9zJ9rK|WzQtJ0Zl{`L+U($V&T zRm-h?m=#*x@y!`A-$ChByYP`N_Z2e>%jkp2?6`hccN{M9P5TMD8d6I@*_YGOXCz)o z$6HRQkhjgbE2AbR#Zgz(KPqmEvgOaDHcw*U?s>8Obc?P(tYhh!DIjL`?;iZVtoo#p zF0ws#G*~%1gDA^-KrbRT$WL>#ny~cqX!s!zTGr>;2tlLS3qF<01FsE}*zr%s>>oux z)={{gtucBYp%#N_|{a<<6;0dUkcyOGDdgo^HQ+SmX6bctY z_a8%>!8s8X6$S6S9bfVC^vurAHeG5`P*ilFq0h?9#MyO}=UJsiODZWT4H4uhDJeyu zd*^kHKhn9=DwQZGc|5u$eN|h=7j^gV%F0Uk!hJ%QPlR1laI__IM@RUdgc6=|#cVoK zhZQ)h74tgpozmtwrO?H-dAI20q(L<@Fzb~e^P0ZYDmV*gvD0_3gW8 z;T|ZM+nNSUS*X8|Z+OF7hSXSEDmuK-$CByb;Nz3D_|zY5tfUWTR4S2KE?-zKKbvgD zr=b}Jk1oD}j+p!bJ+HSH7_hevs~hxlMm_$lnl|^u!I(d~feG7A*utNk|D7P||gYYfQjVr>%+x>bJS( z?CUCpcZ@i))`fb<2rcOb-6XMPNoNgnlE^`Wr*YTdzVHVXFn>SFmGeY3WTVUB7%e}lt6u9DL?@D8SZG#^~p z&U5sazJ0riefI1boOm<%D1WQA7QUd>7)pBa0T-A4hWS|qCZhr)pfkyno=Qgz8a6@D zh5dZ9h~**gMcuEs9CP-H{n}1bm7T678Ae366qxo#wB-axn;h|HAI)&!5ZZR(nsG34a`g^^L+9BXUp+5X8NEEgx#Of9&n0FOHS& zRu96Wc4GP&>FMd8FPs)Oo8RcM9eUN|>&m3vUQQ(iZf#1El83Kv3QGjoMUze5D=X#c z@jc%9@BhXueEYUG)!f|F^r`lq76Kdaa|i~-IOyj=tvNlyuoIz)yB*`lrS21Wb|~1; zLz*;TN6}uoi+;N&B1M0t0Qb=1;CUQxAw&&zQEk6c&sn6qA9DLXo(=Rf0~uu2rvK7s zMGt3Z{yR=oSB&D~qUQTws94+D#xKroFO9^Vbazktc%bCr@4vgh`3-IGDsLF(A+*XA zw7-gk`>6ciLG#Xz!WW?qzaxm?#CWq=N2>E&~i z%D-6ONEbvRsti<`yy~^6sjX$J*blf6;g;Iw5p@yZuCVX5%hhy;{tVfEqkG%i4>|_m zlGbA|hQ~cUJ(3)+Rt`uPP{()riVI3NLoCT5bPogdD#XlcA#Xf>tQ)+Xe&aEJCBBk& z_@e;Z7Z^tKjOMDw6+|~R=Gwtcz0 z-XDqlG$7{Y1e%Zf}*1Hrhfy_INp$&BoBu@?^T$m5L{k)RZXK?Ilrwttwe z$O-Z>&7h(VYV-T~jqzJkDK7>)Qe@%`Lr24+_qKCQWV@fxIhxi@%HASKDQR<&%=DUV zn?6=t(YiAX^I2cKDUSF!7ECzYU2}DDDSKXfd?xcwff}Yk_+w@VX(!du+utt@>dPj0 zf)k-?E>%_&a>&5I=cAPLToNzbGDL`eNFfd@lxY;4+s8rn=!T<>NsoIWlmn*Ur3Q_o z4&am!HtCzY)RT@sCXYmT5wCXjE$XsJr*$hE+XkY zwZe5l$xN>qhexBxz&D@Osi~}Jw9&h8BF7gYs~8G_bSyjO%VZFVw@+IV^5)HV z!N|`RPi<6H*EM{U@ie`?y%O1KZa#cR@X*z5MRKc=DEtc|5HCO)Wvj(pb^G@#f5|Nq zC_8^47=0molrE^=zfjd_@GXU3&Gz{F0BTi6{%b9u0}q#zyUFn;XZ;3XCVWt-ckanvgsB3Gl*F)ojWC&K_Czol1e2IGSJ12 z!oX9EC{FP-&sPAFmbznB^#&Q1u3L(AlOH)ea_f|M!KfM_@;C8@U6EPs}7$?gbND`44t4 zh%J@R@Wg%L7kd>y8phIpy=hl|>=oS-qP}-{EMY^>GtDuN%=+O73-tI>IqN@bV4?jd z-6s$gfGwY34hkj_LwX<4f{nN=mdWqNG@uDd^^z$@gF9O4gkl0sIQHW{;6A-O#qo*7 z+5mk{2M10sEC}qk|DjZB7OW?nsvJtp66L)vG%VHQ_oKWfwW_X;z2fCCY|AF*lQ6yo zpz^~h649Hu;Efb%JIlq%X)<*#nNP;?=hbHmLq4U0Yxncx3_nCpOib)ebi-Ap@1xo$ z4idRktSl{IJgfiEYjiI!d&mr=9XxpxcTk?*-u=D3=ZylR8bS_bCj{&`ppP|Z+WD_?F_{mZrvLy2yMDN(Zc{MZ z@`ds#cNx-$Rq_yUz{vc@&d&vOxJ#Bj%(JJsv@=!v;!Z}H;Q}^X{~CB}yGbqiF5^n) z3_ic4>CLe|X|}whNRy{em^pifSIldOctJx|wd{6Ss|nzmn(yG-)ph|TFsF0b+@aMx z(f!avRJ*vxqa$#HKU=7)rfDdSv35lJ1wl*Aii21gC3m+8L7C1OkI}N%OQEh1in8OU z7gQ$j^yXR_8AgCQ^5jZo>1`c9`W|(neO9_r@TO_L)mdOO2&S$aC*7 zTNApwo)7w0*ZbWlVt@6-Uv=Nj=6A>ZJu!!0{q$Fl00P}F zYWrWFg4-7uAay}OL0}&MYsh74K?1hf1@u&L2F7F-Wphsb-{kZ67yuW79m~G?4M?YI z#64cMX0o&ado&7flX*h-#^ub+c9?DO{>I})|Fy}|($ee~FFJ}MaIOkLZx_62Uw5>c zNXyMFFUO_M?~TfnkOy0CbBt&*E{MTPH`mpDtgk6xx6@G4XOB%%TK@h$>hdU%q85Uj z5yg;>W_=*pb`-VXEAEjc$z}t&Ntn|BXawUpd1|HNp&ia8J?c zJtH$a6J7tib9CTwVf=jW-o0CWiFs+Tkrj!1Z+Z(7rTRa8QdIH3CrJY?#;GRQ)hyX# z*(SF4sx6u4mqdhsaB;l=qPR5g1J>FmNZn z&=3gZwxCZ|C0*=#Ms{{QA`m}&+MNyR{#HY22g*g+Ka)?v>&)7LRIpD_W{$K%p{J}Q z+G%}{B-_MTyJZ!U2;FO)5{T9=cvHkNbkeP}y?Rn532q-jhwFP`)`f1hFkKRYyLo!X zpNx%--C|^%@RDn8EtY-Z==i61iXHw55Q+qhP+^0PXM1qrw~b1yK4Sv=`ufm)Vdh=( zG+fDLYNvN}uW!G1%uI((2QNgYlnH^2*?5w_?DEf8fMPTrZz)(wW&nG{MvZaF z!`+{-30zc9!?5*7n)uT z8j*JQ+2`*u>|~3)S$%{)OH4dBb6b)lK!f;VYo>$Td^ zA_|J!BST2av`bgMpOrQ0+nux3giB1f$Fh8e)?PH+4@i|Jex!>%#9Yq%-Y~XZA&f2z zkRL!75?n;8bf14#Z)2P_zMjAh5dUbpmM1eBe^s^0Q084;0is=uCvKu0)t)=^d)wkY zJUrGBlf_Z)+)B*M%;Z4@qT;wA%-JSOMHxhVrS+hD5cl*fvFLJbO`YtzbQAZN`2C9) z7_Wrf<*n-QO)k?Js*5r0?YQb7+Z=4l0nN#wBdlNb=VdtaufrO7Yf zlCQWsApQ;xHqk?}sF}8o1&iYk;|f;-s7|Y=&Y#A7=vk$w#w$fH-nXsX8lSK)e0AO} zz!KH|I(4!7ynH0Mqz%p(2q_tID? z?SD_e1=XIZ!pQ4yVPjf!lyCef7cs6ho>*UUY9L27q-c0MUQm@v<{^*PyV$+_9xQeE zw}l1xNL_$VT;h6+-YpTfNlAv_{mSY;S=)#2^4@HO1}vhcwuDUM(SS_Oc&lpcR2KpTxj`pAAVQgBZBU&#Ue6jKMO_m4Z6bN+-M^d=n5k@`MIKkCe3Y z0qktxJz($5pr`G}41nHW&`zdR-LgtbK@T5VLCW$j{Cie*zQ-S|T$aJ(Iun6QIn?_$ z8iK&hhGrvySJ!8l8Lff@L>Q1ki=I1-3fV!sUVGzzox+Zd#fWaM6`Q{mSf}sB!gy7~oMnJ!N&tlh?pPvuf{cTanOrpfYO*$M$r1-~erO0zmiGkDt`I#Tqacfg7 zQ`dtXE1p1->=v6<0Qpl3IWVEEJL@q_mooN|PI&m;NL#?SH!fbM8ry@0fGHtE0)5qG z>Ny!3te?(TxEu!@2l#DzL*91Afj*^@!6$Sa*KVVL?}CSN$ODIiKTI&cTA5R{^cyZXVe7YPW_z$BT7KUeV619Q9a=a_k1jWjv&nxk%2#q~9 z@X5lLcBK<2maNXUi%hAddEV1OJ@Y!ZuP$FC|KUA;tZA09n=s8-)m>Bz#Q!GA7h>SB z2`iOg^BM&@#?_H8c|KkdooN(oo8ycSgP6i4^(@~%OwjGrm?QL`e|N3fj8IpVQ zrdYkYLY+|K{?LbF*;v*b#kgu_mq7a1(Uwd$YRWqoaAErxL2Y3rnk(&I-SX%du7^Ddp8uVS`{nCSUUnQZF(r8IyLglP{+nwFUUIz zGOsOGXaJ58<6Z`9TFopH29F~p*jZV{n9cx#*X-X+H`^!hnBg_XLqAuR`X=zKC<(|qy~;&w=I*F5uG{!lLe&PO-ZDT!T_|-x63A4q-2kJo-i`szQk>l@XmfQs(7IzH z+tj1#c4OAS-GP?|$j&5bHMRc5>Y*v}ZVcHSnLenBZ} z7v@Os^VOeWB7*Jd)2>3K)zX#Nr%s4zSG>=Zb=|cJg{BNpZp{G9?+!MIQ+hvpN}WsB zXJ5$fgY^|Xu_XyDsZ1i@g|%#-v=H`wCF9ov+}n{~{h@2>DV(tB*sfZ0=~B$d(#m+X z+;%PBO+&-8Ax*1`LA;z$=g>pp37EWVZpY48h*7$g?)(f z=4yHu+`fM&#=7u}xF?M_q%HzCql8)$ZeqHEEMRa~_oT+iMcw74<8A6Z_3cy97lW6w zk(&;w(5&>*2dsOEIr66wA9a1S@R$$y!Pye-D(|1wJrg_92J!(3qK#Js@|bAblrDSI zm&Esr4UC^%gZ6g#Mg0j2rYVl62cEHq^yC8vr-LI>z}}zgVlO8es}L3%tc+W2eF)aQ zb6@zCz40q;yCW7NU>+2Glz=;`Nm(eEo$Mm<+EB`dn0X@hH^Y0IaLkiqx+!KB_ zXU1`IbN&yXw@0~I?Pc&*rvDs@-1LL{lB{|`Uw3yn?&}j?{=iQ59*LH53(dF80Hjdz zj^8hQ03_g@-QE3!YzrqIk{L?4Fyg0B&=dul{)f5dg}8eWapu%>Bo3AM4mvXMpdfQw zsg<}t#PeYE3}kYX zfWCPw1YZ?hmM{<@!?R!NLcwE5LG&?3`zb$=0mxPKPuz8^PyDw4QxzKRV_4df_U_g(oaTcxqBh1td2QUYPp=hd|jg84>HLqH7rsiEB z)CC`G70y^5I|l&1uD-*7n*&hBC}c)cznl+=$}WmJX)g4m9Hayac6WCjO{46z?vSHo z-hG~atf#w^Dn|3hR}u)+wZD*YQvI^Tsi+B4X%)W{LCwU@C`|}+_t2u>MywWtD~P-e z5lU<0=|ECu<}9Cep+y5AvbIl*hkjiD{e93yce+QqAX-c^Yss(ZFrA6e=lpgKJ9Bdp zqe?Zbk>lUXk)WS1JHOWh$dQ~zDNGVpBBy%>f$RcD0rez_y0%78r0jc}pS+671Qr|B z?5Hd!S6x~8%uaM^tsK~)Tel}-S;W(wc_7>jqlX-3{}lH`!D_w(qUoV-&#zT&yY_56XKpGhfe9w`o}cw{#Q z5RL49{wvR(I^;gU&dk90ScX0n=)~9bnk@lHD+&EAVV(yh(CxP zCE|M*%)=dmJ;mFg=<5UJK1Oe?#xNrNEzU0)t$RY_FCmBiMw}Lo_K2YlsN8J)w5WZh zW0MnmQIDzshy-yZ@!}4@51k8Y#yu*85Ks)!nmw;*CoUGk$nyOAOV)qXnh+HuEsaks zMKs1f$=r5R7797XRso=J)F^U)W+=gFI&A2s;LX(UB{!P{Rdbh{XCDLy_b$>mB>Nlu zzN<_Atl!H4R~~kl%_yTuVwJd6rz;GF`rG8|oTJ!%RFiY`ZplJ@osdU)pH=^xA&aC& zDq12f=cZzE7-eeC&5RXo?;&LV*4&(vmsi4>Qqg}+dlZYEUdbA(5tt-`#0a4JHBuV0 z(luj8)-p#pg**$UGpna*90O&HXQ!wCq68|8P+V2bN^J{=y&94#7#9?Sq0<6!aik5>LNMxMF$_4-flxb91H zIE5}^qx~FRY|sqOoDzGTXb!{Ouy|nr_U2RCibUq>ET!J8kgXy__S5pW{v1cd-x!Sg z&Dy|-)}M!LA}J+>5x$TZN^lIFx6v-uyUryRb$)bg?1o$WAGXP=h9&Wa{_~vyT1NMe zf?Ay+TDE8*F@IZuo$E0Sf9?xr;ss#?Yv*t2pm*gPh@1Q`EIx2OFPNMyK;{i_m)=!v z8>(#S+H(KH$0wn&obRK2+RZ-_h2f8C!aGb&mRz}wD znIx7YZL&yFim%MGhYGY3R;^kjT^%y#~x*!N2pKDHYIkRDO_OQAGCX0Z8fIalCYX{u@u(Q*J zty_JT3PkqYgHFC~DTfPS)Gd(C3FQWn+ja+x9;-Q2r{9VWE6^`nK6fy^4SNlB zec#;N?}7`3nUsEK;x0b$K@3F!VtJfuL(KnAiorygU-p&X0`-5Ae`5k36@qug3jLr=Dy8fi+9p+n)z#WACDL9by051?vBU z8o5OZ-66=nE-Wmh2pVdQe*j(={nyFKvIWY(&4s=QCmz9`B;d_dW#mY3W&&ULG&$zB zqdD+)U=a(W1r{xE9(UKKrkX=ZlX6by^gvKFM=kVDe9gJT`7q_Y3}^&N5vC8ExZKOl zf99L&dE0S{DwQsHQyGqdijVUHNt8hLVy4F+l4BF_tpI}@+#1JX0jq!W(|`buinXQX zOT8B=K^<2mTbqFoDzb>EZ`?K)UXj?50=Rk^*v$I&fq@FclU1O%Y;SKrf6gX*INcha zo}QfzL58hMk?qE105o|0S&ua22321R!=t{?1#K?Sh7WL!n!oS4J7e?%xUo^7n=XQA zMlVQ|r`v)gP@4w>+1dlHG&)JCgv6-+&qMC6nkFd!M}zWE_&XPJ$jEkLWd zxR~Y>{fVob)PWb$1;T;mnQnkJDMHJS4Z@bAEMLT=)B3Cfrcd8rt)Qf29i$_MXuVmy ze?b{24?nHu2)fuM)6akEUn<*@XiXk8t*@_-d#S9$TmX`#x!p!lTZ!PVs+MX=py44Q z*_~1+BrkPJ-kU3VNZ9kKeDm#duC@s0-Xb?W3|Cof;G^S{9d{(9_gper+JU#kAzV@_ zLDo~j;XSjdN7myZ{k1rYM>P&C@m(imnJDfntHsSdr7UZ_kVuxiaX@$Bc!{}AdRBD# z*`R5JNvX;A3yO|Nq^b*LLO){+Ym8lY$wE*Lw^Zi6^`RJV))wYh)Q6f6@;Bq9c$pom zps0^0+x$2^Z=*E3=MB9%Ly?pgoJP|#gBF#q&IgfAG4cpxqg2nBoVE!}4!z&$q59&i zi+gp_(9ZR4{N*30w&?yt?6mXHkT^0|dPP#TFpuOQ8{K%RY`^`w%X6_G zgKg9}kSZlp9zd6(#k=-@f4vta(cBpK`Vs#=)VOc=&Qf_zYcpa6KD}KIUwC++mep7Q z9NpbeJWFV+W9<4M2wa%|2Tzw*AU$4fOYxQ`VPiw8V*ho-(JpG}d|(OWez|!(?-j}F&r}wu|SX`Yj1rz z$%Anave~)}j6#!(ZplV*XI@FnrcvGo&TbnB(l+DS9LA`5(Y3X@_;Q~!ZAaFDl^O)v zwe*@>o#>%cVxR6lc<{^((b|-*SyK7{yIMW9>SV`B^Q|lCn|7=`?_cVRpE*G5gRMwTM`WJxVy%vu$8>r_|2F`^WvZY69W8tGRiIZVrDLb zTcw3{1cZEN@b8WzBXieDf?gg!LZ)b8aZZ76m_snhMEsyNd-vGbomm!|+x00T!QYQ$ z2!!FoFsuuy6@?ApG({$Oq|8?eo1!J+VZp>}uE`0tp|8=8B|bQpN1(?4eesOTzYEqe ziTQaPe%iFj9iNT~75Nu8>_GPZ>9YQ;ZvWpu{G|qPf6{_nb0KlcDZ_=dPEWcAm`RAO z%!S{VzA8pIf3)h~T#URZ(}z49pCS@A`~PR>_lKi%V({xl<*BgxyrV|JuP7El0_w3^sD6h-P{! zJ>+m_1&GeercIS1Vq4or@*{G76|mH#q~Wr_yv@! zn_J=d%IfN>*`_%N{qFAv-iTV}ZN{?9d>B)zw?g0`9hUb9au+O31HxMo4gL>PTuqO@=A3TsrWz?#a8lRYO7A<^VUfxntDy~IIqRweY-(|fxMkpGk&FP z^MrL@4x9R2fF+fMC9y&D+RIyL~67&Xy0v+!bewYbebQ zbkby@c>oo)?t9i099vZ7zIOHR>3=VekB|Ssq+?VI(lNKrVo1q9oL?zK`dJXJ^Iuc< z9n2H9*bkV^4Gph+x5#4q;K*I)c973AF|kkidd6)wge!0No}Fsz7?^A}v(QDgcmD11 zT5+YbL`x=Q#v()3Dogdw^8VrpJh*5gKCnx@b*xYt|JV+cvLTy zV_Bij>fhU}nBQG-;I9+(E=+5*#%)^t(B_gnj1#d?dnE@tCwF_!~HWE z|;NrwX+> zwyTcrIXgcNcYP!sU&(pFL9tw@!%AxncL`3J%$bTw~klwG%f^gjSihulg4 literal 0 HcmV?d00001 diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainparser.png b/metron-platform/metron-parsers-contrib/docs/img/chainparser.png new file mode 100644 index 0000000000000000000000000000000000000000..8fbb2b86d718181eb12faeffbf17f104696b39dd GIT binary patch literal 12069 zcmdUV2UyeD-ewR-M~ciMqn%V=2VMk1Y>dxA%KK$z!H3;GXD!Y`AXN#%o7cVp zpZDFo>=Xil9JtN@6QFq?ML-~`jV2e)+96;IlPZBSbJsf=msP%1P{)c0#+|&2eHdSE zcb*`9<-3feGiBGP%3kGm?-P~UeGnFd{5v)wbgx|XLxTk?gEbGXg8Y;00rV$%j zTFN_cqggv2KYo1w{{5#c#6?Lw1#<+1xirOnQOIw1}*bpe!fiRmxfr8kZ_$l+1c6E0+7mu z`?d%tctnJATqhb@c-vgk@uVv!K2?s@@Q>*NF#1GtJHb zZRp)p3E0^7$B)m;u&GSSyVJ^TtBm#-{b%41T2LEW#y0jA{sxJ>_KfxK!otHtBvi-= z@{r0ASSUj-0F3vF{HAYT6?zsafth6m;*zTsf_1-Sr`vrqCuHP+iUA90d;=cD~WL9mE%Qk z5B%o!Gj#59mJ!@xVscV*seUtOVr-6$Fa{g zll-_;;ppwnOu5L&NUysoS9wbEE64ZMzH&NzqO1DsmSB_x{wGEBIknYL!?b4S)`={g zkOZeBmNc61`8}Jh9!Xv&kB)9~V;+w?*%;mX*}#CDw7Nu}afhp=r>8eO7WJO|2~rta zSIzEOI!4TA+jz^B9K&nz#tVv$#Gbjr%k|bsFs(YYufX50ed_Vl&t{5;6&qObio}V5 z#l;(ojhCI$w0XrX*C!NSLn3VTP_$zv?F@vsv8M5opoVMt&F)e#}>b(&@K>k_?{-dFGq zr0zG;+}QNA7f<#EwtQ2Dwz!tAkww3|2ZBhW+8~V3+nZXe6^Wk}j8k)!jP6NK7>O=A+geuO1&Ht;%z(!+YV*ZEK%PT7W zlQ7 z$<^`mA4awu=-&rCs$Nj|h}{7zXYe02$_<&8Ov=;RdyFyZXH)NaFb@@=nj`l@%5uPC zQQYIgNPAWHPPt455Ejr^&{qCfp5ye|oO_w>Ej)wY3+2J^H!C)|fw~ zZvR9Z`D?egxBEjNm8wD&F7@^GBPw6D`fK#Ip9L|vQL~){wwG4Iwx0!&7P!U{chdB9 z(a~N($mlN&{M!<%yy8nR&f@#`FB7H(AOZPo`-*rXfj|H;Y$7on@={lkXqnQbgch@+ zfyc${#-y$dZr;u&1^jaMLmUY-AvJ-N&yscMwHWl;M(fUo(blZh2qNDwYVCdaJpuQE z&-Mrq@O6$h?rhO_3=9nLd)jN8nj%psR}@U(j_4a72DJVye2LZ!uBFmiT(%e3WToH# z=~v)Zp5)|!h*c$3X#?y_i+odPX>n0ia&>m7O7ZD6e@Q#JXJrc+!%#4~riuCz&jq{l=%+3UGi||Mu zDYD!Kp>yAVv-4=4A*v?{q#1V1r@+iA{qA2ufUJrfxH2AUJAz3+Z1Sx)leKy?HUgTH zm*-%%t^qtgm6;FjO!=;4=+X;;@xsKQGgP?h^O`ARZa z)!W;9nnB4ka#@n-&%7!5^*Fv03Jwns_wn)BPI!?@GCL$!2`}(%dGcktH(gngZl^+L zljB%9_;3LY^RdDrZ;On3JxSgSuR7)Kr}oT?;m(WC`VKa@rX4+U#LMwZj|ZJjSL-Es zAhXqRJ%J4sw_#4wgcDb7SRl8ow61Gm1DAOk>Z(IHexusSzl?Y*_4|`%pCG1Mt}s~I zxV25`n2*M?CkD8-}`@6q0T^wY0XQUPVpAg^(cwK#{(e5nE|=C71Lm|Ij*8%~nX3?4 z;ePVuRxr_Xug>aQUsQqzQdf7X&BB69TjTV6NM0b`ygh?!r5Ozf*)q2_Rx|NDrEii? zFpDKU!BsarK@P@x1YyJ(Ou8FeA@woBAe?P&LcH4aHf+ zCD~3ph{E-H#Bd|6NOhwwc+;lTD-iWaxXoXiyY#)7j!1?Iaj5icTA`l4K3`!~dD@|= zl08<;RQkvyFr_EnK{7{=`XsTI1}!aL=zle{73zk(Gtxh0M=a3)CymbS?da>zqt7dq zrstfjkLCE-4??}x=BO1B^uFY=Y7~L(tf8Ev9-7M7TslJGAKdT2C<2|r9!%WWHVnp= zH)ti#AXrh~*`H$wyeyWqWDH2~m$}5KQvR>>T3p{30m`VhH(R}}o4rO`Lq^cMjGhu3 zAoQhKJip@Uk6Bbh^LiPKYntDe16%l%ECdPMLqECjCgG@&A!^<3kFfSng2$iCbpAgb zeccX0rBY{l8=9Jwh6ecQ)eQF2ry_7{r@NoCzkK;ZYel2kHozI1WG?@>+(jl^@Dv0g z`WFub`S_Ihm(7?BG)Jzgavuq))q~`!bExGNh?M-omt%w2SL@G?Qq%?NQt1T3|CXTm7U3`DGf!4ljBUIerWk!H|<)U0vlag?Q0}R_d2pCp+z{ zacc`3^bX-mm*xV*lsd)@r7YFz#+t0gqR-6QwyPb~7CQ*}{`VT^LylHq2!vzHQ(Xdk z@J_?>!(mK1vC*oSZls-6F<>3X@SUT7(vlJN5g+fH%*%pn0H=^|v(uGBYSPBDPp)A1 zu3TbD)8|$viVuzdHfe-zMMqY&j5of!)O2`d_gF!ah3u%&ya!ExJ3 z53x^pT-Zz5D7$YJr!SwsAdn=g|a3PshRIC&>FyTg1UnAF+=*peU8gHOr0u zZKwyDDsQ*0ZA2uFj1|b74LZ~Lx%6Eu08k|10Q0t3tS51 z5_|#zLn_`KhWx_Nj6026*YfrFd9bHCGOjS-)^R29)sqJ2Trt~9o^N*#Xw@n#A zW$A9&?!D$ZW5#Fg4%wnH)XR0)55*Rm&MV_H6epvfmQ5#W!(2aD3+Vq!5?t5ZpRZFj zWF}lI9PGLNYy9{UcWDHT#fX_o_4^v#=~vLtw_@(flD;Wzi?B@}_0kR%L+u6q4EzWv zbhkm`)FcJ5GRfZ{Nz2K_`NsCsI!Q*bYP!ly$SGHHtlC}O{_ zl0Wl?Kq_4+cM9qCA!~R*1J*m+lODxwXJ>fnN57d6cdr$vIV48G+P66r@G7xpG8-}G7<vw$As`aURp=~U5`J^!PnVMa4;*a0 zDe#RW_^NJFKoA7-<{M1CDZK;Rc7cI312vWyrv5s)E8xTvA}A3qG4QY>$k35)J&5}g zahSa6RBB~1s0y+aiRFdt?adjZomne%691bOd&I`ZM)3Fdo;-OnK0f}5+Y`3V!7k6CZWOfBuvYRes(C#vs*jrw!DPciV^z|JsBXWGg-y>0VFkwf5-=in&8FNx zBOcfxl+75TcSzDA7;O>N$9JEy-(D9AhKSSL6^PevK9CW?bRM=v2(5ZSW0=J zfuMrkR4Qce$)}SxR_3T}ZD+Uf9cu8b?nikN z#ACYsh^dc2{@UL=0kWV)1%3N5Q_63u}Ec)rFUqK<#ck}jbyEut% zf=9nCvw=W1+6EyK^iGMpy|TMJcGg$s2&xX0XYBhwmY9a(q#G_2{|dI%NjUR ztXAd{)e1TKgS<&7PEbco1D$(U2ioK3L=!Eg?{z{va8htHG}sC#?9w>4SAEWgGcm?G ziiRO=LXpG!nxYys<1zJw^8VglAW16dX7^3;GU{3=Nz}QZDt{!l8%<**P&fD!64wqz`Ta*22yWh-Mg2H<=VA z8*Z(F81brwPjf-*ai9P?=dmt0`*LBOeJ;Tn)9a3IPg!o)8`WFEknQFA7DTU^+oEo% zL!5((@iVr4agG9)ej9JJ1$;76vyVF0@OCP-zASD+RV^h`WrwAd&3f0Bw7xDbnMPf$ zrcT{+$|%T!A}CMrhTF%|ALbMwmAx{OOZ7uT!W)MAuB3$@2eZ~$T4u0(f`-E<#bFIO zQ!?cpLsR+`EuMBvN_$tbs#`_{VX?CzPSOJ$~^5;?8;{4qyr6v=ieCm+9tgvdQ8orN17cO|*NP%AL zU#LE56{F_XQnTL>PWFcl!}Zj@neG_gr(qS3gSpM(FsGVVHtnf{wWQ6M(F%1CD?EWHc30pOPPLzHM*rkDVTuai8ols zb9N3nC>Gke_fO?m>Qr)2$?GZoa|o@jCsD5v}POr#{SFs59BCl*8_a&jNtfV8v zC*Ve&!vhZK^b9jovihtsB}=A5NtLJ)ZOS6p=fNLq6gJk{==f=b^uP4iKi|*2Y76um z-K+WBumb)>MdFFR4|l|};X)m&PsCcZqg`Lk+v0nt3^_usIJm(E6ilHS#7A9ZZsl~j z&kCIbKYqkuOhU?;ky@O&bN}qrPq}0`Y}Cy&U1EAY>#>hBVWtO)Mecb-$upLp*xVPw z@ml*#$~JOoM&*`IV>MQGM_k0L^7GWz>KV->5imn99G*({2Eof=Xb zxi|{*U`*F;Zm;lGDRIK)%!^H@WP-L_P(MnE`dpRuUfdp@>kvV~ERW^AIDnjfUGCgy zqhkO+V1HM#hl^Xe8-Wz^r2DI1`@Gw?EXd%zgk?sI{!Hoe#8}Us2Jfk)CJ9QXj+-QB@=AFJ2ynn2QgdnY(7KDS~I#Vs(^rG zCFJHEl&IA?XjuP;ybdw$wqcYn7Z-J@quoLFa>j#HIk+z-o#N{q-4lrb+G8rBN!cLo z_Q|LS<;C$m2H2v^utL|h3z)3R(pjj3*{ z>22p%5R~<*^&4|$uY9S`a({@ltL|=Jk9k5>&1=eG?C|n-8rT--E` zuWuelAP!IUW&|eQ?VAhT|FchYNKn9i%)yp?HmlEGt4n>SpXZl*^&Fg?Y>T%)L4Qla zDI+YKzm#}BAj+K6gq3o<+empj0lBn`I)$v#NPs8KcoavB$yIA!JJHRa#ZYA~1=N+1 z<*GB*B^>=5Iyr*Y!UV=^xH`)9Sd2ZQA<>g{)egHX!W?1_@J?xV!TY^TQtI>bLf6Aw z5!SR|iY<=5%07@)B({D!Vl7T$i}P@cy?(WI&FFc3`RHVD(vIa-%fs{0t?-(82J55! zO5WoQr2I_8TZO!`Ps?SlhA8Bb%~kZy>hh0xN91DPj%4~MF2vfB18<5_$<(R&KfoM2mGs5h19frT!w7vnD}Z3DrPldXDvY*its47wPMry zK0+Plpz;De7VKQP(f(!S2OPmz=eKvh@w2wjSWY)qmhx_ES`BTYzBS&uHNKW;KR>md zejK}eCYe(4+=|n+s7PcvoF>HKoSsfB&ak=7Al5ab~DH5f(}XvyDFDoNtb zK7}`Jf2L;bgZ8K9NP0n23~`iPU-BENOSo>`lSG9Q76@*=Fjom!d|}1V>e($J`jzGg zD<^1cxe@VYLNF>1uHyvG0~>Kg^)K8nkyq;Qdzvq6P1mj%Z8MFsDyT{+N~Xy3$7Zf^ z=Mr-?bVAK)YGF@eSthvs-5S2Q=5MtYRBr!uY+pdgOyzR^u$;VWH{E-Uj96qLu7pJ}^Up*O zF6x$*p%w%2qCFCsD+yL9v2I&$1CE3`?)NAVQrl@pOAT8+7>FCAly3OELM#VFnvPOl z>2s`c8A8An~}m@a%5!hs(M2}8e22&Gr#G!&kieK(9uA1}a}`d}P#a&uWk zm6Ge0RFS!g3~pZ!Y1H8da6 zpZV#h;BFBw=19-Q@`B8~K3fi^gF^b`mvLS}Th}H!wNh6?Z2tjWvYBW9!l=`*ZnsN% z7aXG-9F>ga`=t z-zl^H3sidlt_UE*;4>9Oydks}RAfkqEebYfDK9TSQj)x|3aUg

;eN$UKr&d)#6Q z5X*%>;L1DuLA%c#q~}!7o~*nfR1pB+97k;oRTDHQ%d$yxSpef5wP1u=08D!h0C>fq z8wy%?t^jU(*~r_exE*ANc<}L)m0By3*)9sHbOo#n`_zeS?P>CWHDhFFsD{f83C`RF z2}}n_=hoWtPue!OXB#H~HU$~yPx?24&}TIr4iMG<+Ib`w z6(H1SD+WFd2(=DSNj}&Q@K{_C5NcbIIKs!{cf@J_d?Xy*8okAlNO0h{NdCG;)@w|C zYik}lZw?@LO@ffhLTq_1lg+*fV@ys=c*frKc{e$!lDrF2`Dp4LsE0=!DYrsHQzuI# zXuZS1BK+^gVQ{>ZejZppA~_zpvyIZszm|#tSV?*UXYEK!`rFA#13f)yD`80GThK^1 z{rwI6)~)60rNP6BjkjG{l80b$wYC?JmVXq0=mT8(r^gQ8jK-z}_r-$^@>?YT@)j02 z91cK_M!w$J-*{Gc$jciu+Ii@T8(cV;CW94BM>i$lJspziBcm!|pA5$68p;4|EZQ@ioOfgl@}QLfMZz&}?0I|JZ26n@vA`d&~ds0{$Ei@$U}k&C|KLxx07og1p;*^X?o z->8U|U_k##4hWIRED21EL_0CODo&rDTErHuR-vMB9+X)Al5My~;-MJhX` zUYk2_*N5(s29ES)o$1XQA+@A%eitO{fQmFAD9GRLS-nYbj-)DQC{vPZT%DkUY#_3v z_XxGZjo^@$9>-*{Sbzjkrn2Fk&I-2nyt^KMOuOd`;M;*tFF>Oy@0y#NC!&Ej-NKJ} zIyq(QfTl}`-i~)~_jiv*H; zY-dwU41gYYsy==ZaqE_K+Hv3N<79{X136D?+$U`mpVl}$Ea|Vfa82vEpu&(hM;RHx zNNwsmkLy5hJ{IgxT0iSW^1e46#)UWoD&!HZqtT_~teD%3^E7jySancE;<~nqjkP4W zrv8Tsd`$RFS$)#axd{) zbJ4hv=84>&AD9?lWjnGAU_OXEjunSJt}(6fG;-`G&W`1tlq2FJFxa)$z{6dHPchtSvXZSH3Bh;TVwxDiR*LMCJ~6zJ!I4DLAxZ1dW?7L-d9!oZds|JV{3pqUz;eLw+Fy81e%U7iaI zt0j4j$&I`<&m8n0lQVPfam&bl=R-M3tiH8%*cpGpN0Dd~x$jM%Q?zLXFAZk)52n@q zY3mdY5CRGWNv zEP6$=r;0qt=M?swf1Xe$GNV8BSKBHI9z04AswjkVkJNHpo^ypOR+!<_t}pY9&}j@{ zhai*59yS$I{Wb)PkOO_Z)yZaxJq@i2ZN?sxvPWl6m|hRNi~hAUvzqr16-> zjKC$0VShYtM(n40`LB2V3%KjIQi}ba_PB3*gA-He?fD?{dO50q2?xFU9ilsFanf;4 z{#!R4pn8FEz|C>ynk|4^mfo&68?^$C)8)hyhb?g$56Kf1aR#pI-)4FLop=4mGxi=u cZyJ63Jigd(*rEVX+aV?wEiaUxbN}tX0pLap00000 literal 0 HcmV?d00001 diff --git a/metron-platform/metron-parsers-contrib/pom.xml b/metron-platform/metron-parsers-contrib/pom.xml new file mode 100644 index 0000000000..e016588c2d --- /dev/null +++ b/metron-platform/metron-parsers-contrib/pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + + nl.qsight + metron-parsers-contrib + 1.0.0 + + + + 1.8 + + 0.4.1 + + 19.0 + + 2.4.3 + + + + + org.apache.metron + metron-parsers + ${metron_version} + provided + + + com.google.guava + guava + ${guava_version} + + + junit + junit + 3.8.1 + + + junit + junit + RELEASE + + + com.hubspot.jinjava + jinjava + 2.0.5 + + + joda-time + joda-time + 2.9.9 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${shade_version} + + true + + + + *slf4j* + + + + + + package + + shade + + + true + uber + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + com.google + com.thirdparty.guava + + + + + + storm:storm-core:* + storm:storm-lib:* + org.slf4j.impl* + org.slf4j:slf4j-log4j* + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + true + ${java_version} + -Xlint:unchecked + ${java_version} + true + + + + + \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java new file mode 100644 index 0000000000..9a202d84da --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainlink; + +import nl.qsight.chainparser.ChainParser; +import org.json.simple.JSONObject; + +import java.io.Serializable; +import java.util.Map; + +/** + * A ChainLink is an atomic unit for parsing which has a JSONObject as input and produces a JSONObject as output. + */ +public abstract class ChainLink implements Serializable { + + // The next ChainLink in the DAG (when null, there is no next ChainLink) + private ChainLink next = null; + + /** + * Method for loading the configuration. + * + * @param config The configuration to use. + */ + public void configure(Map config) {} + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param input The JSONObject used as input. + * @return The updated JSONObject. + */ + public abstract JSONObject parse(JSONObject input); + + /** + * Get the next link of the ChainLink DAG. + * + * @return The next ChainLink. + * @see ChainParser + */ + public ChainLink getNextLink() { + return next; + } + + /** + * Set the next ChainLink in the ChainLink DAG. + * + * @param next The next ChainLink. + * @see ChainParser + */ + public void setNextLink(ChainLink next) { + assert next != this : "The next link must no equal the current link."; + this.next = next; + } + + /** + * Determines whether the DAG has a next ChainLink. + * + * @return True when there is a next ChainLink, false otherwise. + */ + public boolean hasNextLink() { + return this.next != null; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java new file mode 100644 index 0000000000..f8bc727016 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainlink; + +import nl.qsight.common.Constants; +import org.json.simple.JSONObject; + +/** + * A special ChainLink that works with an input and one or more outputs. If the output is a JSONObject, then the output + * is automatically merged with the current state. Otherwise, a special Constants.OUTPUT_MARKER field is created + * containing the output. The type of the input should be specified when creating new classes. The field specified + * by the Constants.INPUT_MARKER is casted to the desired input type. + * + * The RenderLink is capable of transforming multiple fields into a single input. Therefore, a combination of the + * RenderLink and the ChainLinkIO is capable of transforming one or more inputs to one or more outputs and covers + * all possible usages. + * + * @see ChainLink + */ +public abstract class ChainLinkIO extends ChainLink { + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param input The JSONObject used as input. + * @return The updated JSONObject. + */ + public abstract Object parseInputField(T input); + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param data The JSONObject used as input. + * @return The updated JSONObject. + */ + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + String field = Constants.INPUT_MARKER; + if (!data.containsKey(field)) field = Constants.ORIGINAL_STRING; + if (!data.containsKey(field)) { + throw new IllegalStateException("Field \"" + field + "\" not found in the state."); + } + T input = (T) data.get(field); + Object outputObject = this.parseInputField(input); + if (outputObject instanceof JSONObject) { + JSONObject output = (JSONObject)outputObject; + for (Object keyObject : output.keySet()) { + String key = (String) keyObject; + data.put(key, output.get(keyObject)); + } + } else { + data.put(Constants.OUTPUT_MARKER, outputObject); + } + + // Clean up the input data + data.remove(Constants.INPUT_MARKER); + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java new file mode 100644 index 0000000000..57f458aecf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainparser; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.common.Constants; +import nl.qsight.links.fields.IdentityLink; +import nl.qsight.utils.ConfigUtils; +import org.apache.metron.parsers.BasicParser; +import org.json.simple.JSONObject; + +import java.io.UnsupportedEncodingException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static nl.qsight.common.Constants.ORIGINAL_STRING; +import static nl.qsight.common.Constants.TIMESTAMP; + +/** + * The ChainParser is a composable unit consisting of a DAG of ChainLink objects. It points to the first ChainLink of + * the ChainLink DAG and each ChainLink points towards the next ChainLink item in the DAG. + */ +public class ChainParser extends BasicParser { + + // The first ChainLink of the ChainLink DAG + private ChainLink link; + // The encoding of the messages + private String encoding; + + // Pre link hook + private Method preLinkHook = null; + private Object preLinkInvoker = null; + + // Post link hook + private Method postLinkHook = null; + private Object postLinkInvoker = null; + + /** + * Initializing the ChainParser. + */ + public ChainParser() { + this.link = new IdentityLink(); + this.setEncoding("UTF-8"); + } + + /** + * An empty initialization function which is required by the BasicParser class. + */ + @Override + public void init() { + + } + + /** + * Set the initial ChainLink of the ChainLink DAG. + * + * @param link The initial ChainLink. + */ + public void setInitialLink(ChainLink link) { + this.link = link; + } + + /** + * Get the initial ChainLink of the ChainLink DAG. The default ChainLink DAG consists of one IdentityLink which + * copies the input to the output. + * + * @return The initial ChainLink. + */ + public ChainLink getInitialLink() { + return this.link; + } + + /** + * Configuration for the ChainParser. + * + * @param map A mapping from configuration items to configuration values. + */ + @Override + @SuppressWarnings("unchecked") + public void configure(Map map) { + map = ConfigUtils.compile(map); + ChainLink linkObject = ConfigUtils.getRootLink(map); + this.setInitialLink(linkObject); + } + + public void setPreLinkHook(Method hook, Object invoker) { + this.preLinkHook = hook; + this.preLinkInvoker = invoker; + } + + public void setPostLinkHook(Method hook, Object invoker) { + this.postLinkHook = hook; + this.postLinkInvoker = invoker; + } + + private void executePreLinkHook(Object... args) { + if (this.preLinkHook == null || this.preLinkInvoker == null) return; + try { + this.preLinkHook.invoke(this.preLinkInvoker, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("The post link is not executable."); + } + } + + private void executePostLinkHook(Object... args) { + if (this.postLinkHook == null || this.postLinkInvoker == null) return; + try { + this.postLinkHook.invoke(this.postLinkInvoker, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("The post link is not executable."); + } + } + + /** + * Parse a given string represented as a list of bytes into a list containing a single JSON object. + * + * @param bytes The byte representation of the input. + * @return A list containing a single JSON object which is the parsed representation of the input. + */ + @Override + @SuppressWarnings("unchecked") + public List parse(byte[] bytes) { + // Try to decode the bytes + String decodedMessage; + try { + decodedMessage = new String(bytes, this.getEncoding()); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Unsupported encoding."); + } + + // Put the bytes conversion and current date into an initial JSON object + JSONObject state = new JSONObject(); + Instant instant = Instant.now(); + long timestamp = instant.toEpochMilli(); + state.put(ORIGINAL_STRING, decodedMessage); + state.put(TIMESTAMP, timestamp); + + // Execute the parser DAG (Directed Acyclic Graph) + ChainLink link = this.link; + + this.executePreLinkHook(link); + state = this.executeLink(link, state); + this.executePostLinkHook(link); + + // Iterate through the DAG until the end has reached + while (link.hasNextLink()) { + link = link.getNextLink(); + this.executePreLinkHook(link); + state = this.executeLink(link, state); + this.executePostLinkHook(link); + } + + // Clean up the output marker + if (state.containsKey(Constants.OUTPUT_MARKER)) { + state.remove(Constants.OUTPUT_MARKER); + } + + // Create a list containing a single item + List resultSet = new ArrayList<>(); + resultSet.add(state); + + // Return the result + return resultSet; + } + + /** + * Execute a ChainLink and return the result. + * + * @param link ChainLink to execute. + * @param state Input data. + * @return The output of the ChainLink. + */ + private JSONObject executeLink(ChainLink link, JSONObject state) { + String linkName = link.getClass().getCanonicalName(); + + // Execute the link + state = link.parse(state); + + // Check the validness of the state after executing the link + if (!state.containsKey(ORIGINAL_STRING)) + throw new IllegalStateException("The state does not contain the \"original_string\" field after executing " + + linkName + "."); + if (!state.containsKey(TIMESTAMP)) + throw new IllegalStateException("The state does not contain the \"timestamp\" field after executing " + + linkName + "."); + + // Clean up the input field when it still exists + if (state.containsKey(Constants.INPUT_MARKER)) { + state.remove(Constants.INPUT_MARKER); + } + + return state; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java new file mode 100644 index 0000000000..8d2d9f2f3b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.common; + +public class Constants { + + public static String INPUT_MARKER = ""; + public static String OUTPUT_MARKER = ""; + public static String TIMESTAMP = "timestamp"; + public static String ORIGINAL_STRING = "original_string"; + public static String AUTOGENERATED_LINK = "_AUTOLINK_"; + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java new file mode 100644 index 0000000000..2478e1d4c3 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.List; +import java.util.Map; + +/** + * A link which blacklists keys. + */ +public class BlacklistLink extends ChainLink { + + private List fields; + + public void configure(Map config) { + this.fields = null; + if (config.containsKey("fields")) { + assert config.get("fields") instanceof List; + this.setFields((List) config.get("fields")); + } + } + + /** + * Get the fields. + * + * @return The fields. + */ + public List getFields() { + return fields; + } + + /** + * Set the fields. + * + * @param fields The fields which will be blacklisted. + */ + public void setFields(List fields) { + this.fields = fields; + } + + /** + * Whitelist all the specified fields and filter out fields not in the list. + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getFields() == null) throw new IllegalStateException("The blacklisted fields should be specified."); + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + if (!this.getFields().contains(key)) { + result.put(keyObject, data.get(keyObject)); + } + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java new file mode 100644 index 0000000000..1f3de8475b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import org.json.simple.JSONObject; + +/** + * The IdentityLink returns a copy of the input as output. + */ +public class IdentityLink extends ChainLink { + + /** + * Parse a JSONObject using the IdentityLink class. + * + * @param input The JSONObject used as input. + * @return The same JSONObject. + */ + @Override + public JSONObject parse(JSONObject input) { + return input; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java new file mode 100644 index 0000000000..8b2eb6e6ce --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.utils.StringUtils; +import org.json.simple.JSONObject; + +/** + * A link for normalizing field names. + */ +public class NormalizeFieldLink extends ChainLink { + + /** + * + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + String newKey = StringUtils.normalize(key); + // Remove trailing and beginning underscore + if (newKey == null || newKey.length() == 0) { + newKey = null; + } + if (result.containsKey(newKey)) throw new IllegalStateException("Duplicate normalized keys."); + result.put(newKey, data.get(keyObject)); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java new file mode 100644 index 0000000000..c4d0759618 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.Map; + +/** + * A link which renames keys. + */ +public class RenameLink extends ChainLink { + + private Map renames; + + /** + * Get the renames. + * + * @return All the renames. + */ + public Map getRenames() { + return renames; + } + + /** + * Set the renames. + * + * @param renames The key-value pairs where the keys are to original keys and the values are the field names after + * the renames. + */ + public void setRenames(Map renames) { + this.renames = renames; + } + + @SuppressWarnings("unchecked") + public void configure(Map config) { + if (config.containsKey("rename")) { + assert config.get("rename") instanceof Map; + this.setRenames((Map) config.get("rename")); + } + } + + /** + * Rename fields using the rename rules. + * + * @param data Input data. + * @return Data with renamed keys. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getRenames() == null || this.getRenames().size() == 0) + throw new IllegalStateException("No renames specified"); + + JSONObject store = new JSONObject(); + for (String key : this.getRenames().keySet()) { + if (data.containsKey(key)) { + store.put(key, data.get(key)); + } + } + for (String key : this.getRenames().keySet()) { + if (store.containsKey(key) && data.containsKey(key)) { + String outputKey = getRenames().get(key); + data.put(outputKey, store.get(key)); + data.remove(key); + } + } + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java new file mode 100644 index 0000000000..26ace1aa34 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import com.google.common.collect.Maps; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.interpret.RenderResult; +import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; +import nl.qsight.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.io.Serializable; +import java.time.Year; +import java.util.List; +import java.util.Map; + +/** + * A link for rendering a template to a new field. + */ +public class RenderLink extends ChainLink { + // The fields and constants to concatenate + private String template; + + // The name of the field to store the output in + private String outputField; + + // The used variables + private List variables; + + /** + * Get the template. + * + * @return The template. + */ + public String getTemplate() { + return template; + } + + /** + * Set the template. + * + * @param template The Jinja template. + */ + public void setTemplate(String template) { + this.template = template; + } + + /** + * Get the output field. + * + * @return The field in which the output is stored. + */ + public String getOutputField() { + return outputField; + } + + /** + * Set the output field. + * + * @param field The field in which the output is stored. + */ + public void setOutputField(String field) { + this.outputField = field; + } + + public void setVariables(List variables) { this.variables = variables; } + + public void configure(Map config) { + this.variables = null; + if (config.containsKey("template")) { + assert config.get("template") instanceof String; + this.setTemplate((String) config.get("template")); + } + if (config.containsKey("variables")) { + assert config.get("variables") instanceof List; + this.setVariables((List) config.get("variables")); + } + if (config.containsKey("output")) { + assert config.get("output") instanceof String; + this.setOutputField((String) config.get("output")); + } + } + + /** + * Parse the JSON object. All keys of the input become available in the template. For example, if the input + * contains {"user": "me"}, then the template "{{user}}" is rendered as "me". + * + * @param data The input data. + * @return The output data in which the output field (specified by the setOutputField method) is filled with the + * rendered version of the template (specified by the setTemplate method). + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getTemplate() == null) throw new IllegalStateException("No template specified."); + if (this.getOutputField() == null) throw new IllegalStateException("No output field specified."); + + // Loop in O(n) time through the data and replace variables when found + String template = this.getTemplate(); + + boolean inVariable = false; + String variableName = ""; + String buffer = ""; + int bracketOpenCount = 0; + int bracketCloseCount = 0; + for (int i = 0; i < template.length(); i++) { + char c = template.charAt(i); + + if (c == '{') { + bracketOpenCount += 1; + } else { + bracketOpenCount = 0; + } + + if (c == '}') { + if (inVariable) { + inVariable = false; + variableName = variableName.trim(); + buffer = buffer.substring(0, buffer.length() - 2); + + // Substitute the variable + if (variableName.equals("year")) { + buffer += Year.now().toString(); + } else { + if (data.containsKey(variableName)) { + buffer += data.get(variableName); + } + } + + // Clear the variable name + variableName = ""; + } + bracketCloseCount += 1; + } else { + bracketCloseCount = 0; + } + + if (inVariable) { + variableName += c; + } else { + buffer += c; + } + + if (bracketOpenCount == 2) { + inVariable = true; + } + + if (bracketCloseCount == 2) { + buffer = buffer.substring(0, buffer.length() - 2); + } + } + + data.put(this.getOutputField(), buffer); + + // Return the result + return data; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java new file mode 100644 index 0000000000..4990a2f487 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.common.Constants; +import org.json.simple.JSONObject; + +import java.util.Map; + +/** + * A link which selects a field an puts it into the input marker field. + */ +public class SelectLink extends ChainLink { + + private String field; + + /** + * Get the field to select. + * + * @return The field. + */ + public String getField() { + return field; + } + + /** + * Set the field to select. + * + * @param field The field to select. + */ + public void setField(String field) { + this.field = field; + } + + public void configure(Map config) { + if (config.containsKey("field")) { + assert config.get("field") instanceof String; + this.setField((String) config.get("field")); + } + } + + /** + * Copy the selected field to select to the input marker field. + * + * @param data Input data. + * @return Data with an additional input marker field. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getField() == null) throw new IllegalStateException("The field to select should be specified."); + + data.put(Constants.INPUT_MARKER, data.get(this.getField())); + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java new file mode 100644 index 0000000000..56ae7157fa --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.utils.StringUtils; +import org.json.simple.JSONObject; + +/** + * A link for normalizing field names. + */ +public class TrimValueLink extends ChainLink { + + /** + * + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + Object valueObject = data.get(keyObject); + if (valueObject instanceof String) { + String value = (String) valueObject; + String newValue = value.trim(); + result.put(keyObject, newValue); + } + else { + result.put(keyObject, valueObject); + } + } + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java new file mode 100644 index 0000000000..d78d23193d --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.List; +import java.util.Map; + +/** + * A link which whitelists keys. + */ +public class WhitelistLink extends ChainLink { + + private List fields; + + public void configure(Map config) { + this.fields = null; + if (config.containsKey("fields")) { + assert config.get("fields") instanceof List; + this.setFields((List) config.get("fields")); + } + } + + /** + * Get the fields. + * + * @return The fields. + */ + public List getFields() { + return fields; + } + + /** + * Set the fields. + * + * @param fields The fields which will be whitelisted. + */ + public void setFields(List fields) { + this.fields = fields; + } + + /** + * Whitelist all the specified fields and filter out fields not in the list. + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getFields() == null) throw new IllegalStateException("The whitelisted fields should be specified."); + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + if (this.getFields().contains(key)) { + result.put(keyObject, data.get(keyObject)); + } + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java new file mode 100644 index 0000000000..9cfac8990f --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.chainlink.ChainLinkIO; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.util.Map; + +/** + * A link for decoding JSON. + */ +public class JSONDecoderLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + JSONParser parser = new JSONParser(); + JSONObject result; + try { + result = flatten((JSONObject) parser.parse(input)); + } catch (ParseException e) { + throw new IllegalStateException("Could not parse JSON in the message."); + } + + return result; + } + + @SuppressWarnings("unchecked") + private JSONObject flatten(JSONObject original) { + JSONObject result = new JSONObject(); + for (Object key : original.keySet()) { + Object value = original.get(key); + if (value instanceof JSONObject) { + JSONObject subjson = flatten((JSONObject) value); + for (Object subitem : subjson.entrySet()) { + Map.Entry entry = (Map.Entry) subitem; + String new_key = key + "." + entry.getKey(); + result.put(new_key, entry.getValue()); + } + } else { + result.put(key, value); + } + } + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java new file mode 100644 index 0000000000..1084932faf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.chainlink.ChainLinkIO; +import nl.qsight.utils.StringUtils; +import org.json.simple.JSONObject; + +import java.util.Map; +import java.util.regex.Pattern; + +/** + * A link for splitting key-value pairs in strings. + */ +public class KeyValueLink extends ChainLinkIO { + + private String keyValueDelimiter; + private String pairDelimiter; + private String validKeyChars; + + public String getKeyValueDelimiter() { + return keyValueDelimiter; + } + + public void setKeyValueDelimiter(String keyValueDelimiter) { + this.keyValueDelimiter = keyValueDelimiter; + } + + public String getPairDelimiter() { + return pairDelimiter; + } + + public void setPairDelimiter(String pairDelimiter) { + this.pairDelimiter = pairDelimiter; + } + + public String getValidKeyChars() { + return validKeyChars; + } + + public void setValidKeyChars(String validKeyChars) { + this.validKeyChars = validKeyChars; + } + + public void configure(Map config) { + if (config.containsKey("pair_delimiter")) { + assert config.get("pair_delimiter") instanceof String; + this.setPairDelimiter((String) config.get("pair_delimiter")); + } + if (config.containsKey("key_value_delimiter")) { + assert config.get("key_value_delimiter") instanceof String; + this.setKeyValueDelimiter((String) config.get("key_value_delimiter")); + } + if (config.containsKey("valid_key_characters")) { + assert config.get("valid_key_characters") instanceof String; + this.setValidKeyChars((String) config.get("valid_key_characters")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + if (this.getKeyValueDelimiter() == null) throw new IllegalStateException("The key-value delimiter is not set."); + if (this.getPairDelimiter() == null) throw new IllegalStateException("The pair delimiter is not set."); + if (this.getValidKeyChars() == null) throw new IllegalStateException("The valid key characters are not set."); + + JSONObject result = new JSONObject(); + Map pairs = StringUtils.parseKeyValuePairs(input, this.getKeyValueDelimiter(), + this.getPairDelimiter(), this.getValidKeyChars()); + for (String key : pairs.keySet()) { + if (key != null && !key.equals("null")) { + result.put(key, pairs.get(key)); + } + } + return result; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java new file mode 100644 index 0000000000..efc6625686 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.chainlink.ChainLinkIO; +import nl.qsight.utils.StringUtils; +import org.json.simple.JSONObject; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexLink extends ChainLinkIO { + + private Map selector; + private String pattern; + + @SuppressWarnings("unchecked") + public void setSelector(Map selector) { + this.selector = selector; + } + + public Map getSelector() { + return this.selector; + } + + public String getPattern() { + return pattern; + } + + public void setPattern(String pattern) { + this.pattern = pattern; + } + + public void configure(Map config) { + if (config.containsKey("pattern")) { + assert config.get("pattern") instanceof String; + this.setPattern((String) config.get("pattern")); + } + if (config.containsKey("selector")) { + assert config.get("selector") instanceof Map; + this.setSelector((Map) config.get("selector")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + Pattern pattern = Pattern.compile(this.pattern); + Matcher matcher = pattern.matcher(input); + + JSONObject result = null; + + if (matcher.find()) { + result = new JSONObject(); + + for (String selectorKey : this.selector.keySet()) { + result.put(selectorKey, ""); + Object positionObject = this.selector.get(selectorKey); + + int position = StringUtils.toInteger(positionObject); + boolean isPositionSet = StringUtils.isNumerical(positionObject); + if (!isPositionSet) throw new IllegalStateException("Position is not numerical."); + + String value = matcher.group(position); + if (value != null) result.put(selectorKey, value); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java new file mode 100644 index 0000000000..63b6442182 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.chainlink.ChainLinkIO; +import nl.qsight.utils.StringUtils; +import org.json.simple.JSONObject; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * A link for splitting strings. + */ +public class SplitLink extends ChainLinkIO { + + private String delimiter; + private boolean delimiterIsRegex; + private Map selector; + + public String getDelimiter() { + return delimiter; + } + + public boolean isDelimiterRegex() { + return this.delimiterIsRegex; + } + + public void setDelimiter(String delimiter) { + this.setDelimiter(delimiter, false); + } + + public void setDelimiter(String delimiter, boolean isRegex) { + this.delimiter = delimiter; + this.delimiterIsRegex = isRegex; + } + + @SuppressWarnings("unchecked") + public void setSelector(Map selector) { + this.selector = selector; + } + + public Map getSelector() { + return this.selector; + } + + public void configure(Map config) { + if (config.containsKey("delimiter")) { + assert config.get("delimiter") instanceof String; + this.setDelimiter((String) config.get("delimiter")); + } + if (config.containsKey("selector")) { + assert config.get("selector") instanceof Map; + this.setSelector((Map) config.get("selector")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + if (this.getDelimiter() == null) throw new IllegalStateException("Delimiter is not set."); + if (this.getDelimiter() == null) throw new IllegalStateException("Delimiter RegEx boolean is not set."); + if (this.getSelector() == null) throw new IllegalStateException("Selector is not set."); + + String delimiter = this.getDelimiter(); + if (!this.isDelimiterRegex()) { + delimiter = Pattern.quote(delimiter); + } + + String[] parts = input.split("(" + delimiter + ")"); + + JSONObject result = new JSONObject(); + for (Object positionObject : this.selector.keySet()) { + int position = StringUtils.toInteger(positionObject); + boolean isPositionSet = StringUtils.isNumerical(positionObject); + if (!isPositionSet) throw new IllegalStateException("Position is not numerical."); + if (position < 0) { + position += parts.length; + } + if (position < parts.length) { + String positionLabel = (String) this.selector.get(positionObject); + String value = parts[position]; + result.put(positionLabel, value); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java new file mode 100644 index 0000000000..97788ebcc6 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java @@ -0,0 +1,294 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.chainlink.ChainLinkIO; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.json.simple.JSONObject; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static nl.qsight.common.Constants.TIMESTAMP; + +public class TimestampLink extends ChainLinkIO { + + private List patterns; + private Map> predefined_patterns; + + @SuppressWarnings("unchecked") + public void configure(Map config) { + this.initPredefinedPatterns(); + + if (config.containsKey("patterns")) { + assert config.get("patterns") instanceof List; + this.setPatterns((List) config.get("patterns")); + } + } + + public void initPredefinedPatterns() { + Map> patterns = new HashMap<>(); + patterns.put("timestamp", Arrays.asList("([0-9]{10,13})", "t")); + this.setPredefinedPatterns(patterns); + } + + @SuppressWarnings("unchecked") + private void addTimestampToMessage(JSONObject message, Long timestamp) { + message.put(TIMESTAMP, timestamp); + } + + private String extractLocalTime(String regex_result, String timestamp_mapping, String mapping_format) { + DateTimeFormatter dtf = DateTimeFormat.forPattern(timestamp_mapping); + DateTimeFormatter df = DateTimeFormat.forPattern(timestamp_mapping); + DateTime temp = df.withOffsetParsed().parseDateTime(regex_result); + DateTimeZone theZone = temp.getZone(); + DateTime dt = dtf.withZone(theZone).parseDateTime(regex_result); + DateTimeFormatter fmt = DateTimeFormat.forPattern(mapping_format); + return fmt.print(dt); + } + + private long regexToTimestamp(String regex_result, String timestamp_mapping) { + try { + long result; + if (timestamp_mapping.equals("t")) { + result = Long.parseLong(regex_result); + } else { + DateTimeFormatter dtf = DateTimeFormat.forPattern(timestamp_mapping); + DateTime dt = dtf.withZoneUTC().parseDateTime(regex_result); + result = dt.getMillis(); + } + + // Convert to ms + if (String.valueOf(result).length() < 13) result *= 1000; + + return result; + } catch (Exception e) { + throw new IllegalStateException("A pattern (" + regex_result + ") was found, but could not be mapped using the following mapping: " + timestamp_mapping); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + JSONObject message = new JSONObject(); + + // Initiate objects + Date datetime = null; + String mapping_format = null; + DateFormat format = null; + String formatted_datetime = null; + int timezone_available = 0; + + // In the remainder of the code, a loop will run through the configured patterns in order to regex for the + // proper elements of the logline, and create output based on the regex result(s). + for (List pattern_mapping_object : this.getPatterns()) { + String regex_pattern; + String timestamp_mapping; + Object selection_index; + + // A pattern is a tuple (regex, mapping, indexing) or (predefined_pattern, indexing) where the predefined + // pattern is defined in the constructor of this class and also consists of a (regex, mapping) tuple. + if (pattern_mapping_object == null) { + throw new IllegalStateException("Pattern type not recognized."); + } + + // The pattern should consist of either 2 or 3 elements + if (pattern_mapping_object.size() == 2) { + // If 2 elements, then the pattern is a (predefined_pattern, indexing) tuple + assert pattern_mapping_object.get(0) instanceof String; + assert pattern_mapping_object.get(1) instanceof String || pattern_mapping_object.get(1) instanceof Integer; + List predefined_pattern = this.getPredefinedPatterns().get(pattern_mapping_object.get(0)); + regex_pattern = (String) predefined_pattern.get(0); + timestamp_mapping = (String) predefined_pattern.get(1); + selection_index = pattern_mapping_object.get(1); + } else if (pattern_mapping_object.size() == 3) { + // If 2 elements, then the pattern is a (regex, mapping, indexing) tuple + assert pattern_mapping_object.get(0) instanceof String; + assert pattern_mapping_object.get(1) instanceof String; + assert pattern_mapping_object.get(2) instanceof String || pattern_mapping_object.get(2) instanceof Integer; + regex_pattern = (String) pattern_mapping_object.get(0); + timestamp_mapping = (String) pattern_mapping_object.get(1); + selection_index = pattern_mapping_object.get(2); + } else { + throw new IllegalStateException("Pattern does not contain 2 or 3 elements."); + } + + // Compile the pattern and try to find matches + Pattern compiled_pattern = Pattern.compile(regex_pattern); + Matcher matcher = compiled_pattern.matcher(input); + + // Concatenate all matches (separated with spaces), so afterwards configured indexing criteria can be used + // (e.g. first, last, oldest, newest). + List results = new ArrayList<>(); + while (matcher.find()) { + StringBuilder regex_match = new StringBuilder(); + for (int group = 1; group <= matcher.groupCount(); group++) { + String item = matcher.group(group); + regex_match.append(" ").append(item); + } + if (regex_match.length() > 0) { + regex_match = new StringBuilder(regex_match.substring(1)); + } + results.add(regex_match.toString()); + } + + // Try to convert the found patterns to timestamps + List timestamps = new ArrayList<>(); + for (String result : results) { + timestamps.add(regexToTimestamp(result, timestamp_mapping)); + } + + // Order the found timestamps according to the selection criteria + int selected_timestamp = -1; + if (timestamps.size() == 0) { + // No timestamps found, continue to the next pattern + continue; + } else if (timestamps.size() == 1) { + // There is only one timestamp, so select the first element + selected_timestamp = 0; + addTimestampToMessage(message, timestamps.get(0)); + } else { + // Advanced selection criteria + if (selection_index instanceof Integer) { + // Select the (select_index)th element + int selection_index_integer = (int) selection_index; + if (selection_index_integer < results.size()) { + addTimestampToMessage(message, timestamps.get(selection_index_integer)); + selected_timestamp = selection_index_integer; + } + } else if (selection_index instanceof String) { + // If it is a string, select by newest or by oldest + if (selection_index.equals("newest")) { + // Find the newest timestamp + long max_timestamp = Collections.max(timestamps); + selected_timestamp = timestamps.indexOf(max_timestamp); + addTimestampToMessage(message, max_timestamp); + } else if (selection_index.equals("oldest")) { + // Find the oldest timestamp + long min_timestamp = Collections.min(timestamps); + selected_timestamp = timestamps.indexOf(min_timestamp); + addTimestampToMessage(message, min_timestamp); + } else { + // Unknown selection criteria (if it is a string) + throw new IllegalStateException("The selection index should be either \"newest\" or \"oldest\"."); + } + } + } + + if (selected_timestamp != -1) { + // Scenario 2: There is a timestamp which has been selected + String orig_regex = results.get(selected_timestamp); + timezone_available = 0; + if (timestamp_mapping.contains("X") || timestamp_mapping.contains("Z")) { + timezone_available = 1; + } + message.put("timezone_available", timezone_available); + + // if datetime found is an epoch timestamp, length is checked + if (timestamp_mapping.equals("t")) { + long timestamp_epoch = Long.parseLong(orig_regex); + if (String.valueOf(timestamp_epoch).length() < 13) { + timestamp_epoch *= 1000; + } + // timezone_available set to 1, in order to avoid that the fieldtransformation adds the + // probe-timezone and converts accordingly to timestamp, since any epoch timestamp is assumed to + // be UTC. + timezone_available = 1; + message.put("timezone_available", timezone_available); + + // formatted_datetime + datetime = new Date(timestamp_epoch); + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + format = new SimpleDateFormat(mapping_format); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + formatted_datetime = format.format(datetime); + + // since the datetime found is no epoch timestamp, there are two possible mapping_formats that should be + // used to normalize the datetime output. + } else { + if (timezone_available == 0) { + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS"; + } else { + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + } + // formatted_datetime + formatted_datetime = extractLocalTime(orig_regex, timestamp_mapping, mapping_format); + } + } + } + + if(formatted_datetime == null) { + // The script starts with (fallback) scenario 1 in which case a timestamp is not found/missing. In such a + // situation, datetime (=the variable used for fieldtransformation(s) in the config file of the parser) is set + // to current time. + datetime = new Date(); + + // First, datetime is formatted to a datetime with timezone "+0000" (in order to make sure that a timezone is + // always available during later fieldtransformation configuration in the config file of the parser). + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + format = new SimpleDateFormat(mapping_format); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + formatted_datetime = format.format(datetime); + + // lastly for scenario 1, timezone_available is set to 1, so fieldtransformation configuration will not append + // the datetime field before converting it to epoch timestamp (see the config file of a parser). + timezone_available = 1; + } + + // The code above produced a (formatted) datetime, a mapping_format and a timezone_available variable, all for + // proper use of fieldtransformation(s) to produce the proper normalised epoch time. + message.put("datetime", formatted_datetime); + message.put("mapping", mapping_format); + message.put("timezone_available", timezone_available); + + // Lastly, original_timestamp will be set as a non-timezone version of the formatted(datetime). + String original_datetime_output_mapping = "yyyy-MM-dd HH:mm:ss.SSS"; + String original_timestamp; + if (original_datetime_output_mapping.equals(mapping_format)) { + original_timestamp = formatted_datetime; + } else { + original_timestamp = extractLocalTime(formatted_datetime, mapping_format, original_datetime_output_mapping); + } + message.put("original_timestamp", original_timestamp); + + return message; + } + + public void setPatterns(List patterns) { + this.patterns = patterns; + } + + public List getPatterns() { + return this.patterns; + } + + public void setPredefinedPatterns(Map> predefined_patterns) { + this.predefined_patterns = predefined_patterns; + } + + public Map> getPredefinedPatterns() { + return this.predefined_patterns; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java new file mode 100644 index 0000000000..3c1b4527ad --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.common.Constants; +import nl.qsight.links.fields.RenderLink; +import nl.qsight.links.fields.SelectLink; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ConfigUtils { + + public static Map compile(Map config) { + config = unfoldInput(config); + return config; + } + + @SuppressWarnings("unchecked") + public static Map unfoldInput(Map config) { + assert config.containsKey("chain"); + assert config.get("chain") instanceof List; + List links = (List) config.get("chain"); + + assert config.containsKey("parsers"); + assert config.get("parsers") instanceof Map; + Map linksConfig = (Map) config.get("parsers"); + + List unfoldedLinks = new ArrayList<>(); + + int autolinkIndex = 0; + for (Object link : links) { + String linkName = (String) link; + assert linksConfig.containsKey(linkName); + assert linksConfig.get(linkName) instanceof Map; + Map linkConfig = (Map) linksConfig.get(linkName); + + if (linkConfig.containsKey("input")) { + // Unfold it + String renderName = Constants.AUTOGENERATED_LINK + autolinkIndex; + autolinkIndex += 1; + unfoldedLinks.add(renderName); + Map renderConfig = new HashMap<>(); + + // Figure out which variables are being used to speed up the render link + String template = (String) linkConfig.get("input"); + List variables = new ArrayList<>(); + Pattern pattern = Pattern.compile("\\{\\{\\s*([^} |.:]+)}"); + Matcher matcher = pattern.matcher(template); + while (matcher.find()) { + String variable = matcher.group(1); + variables.add(variable); + } + + renderConfig.put("class", RenderLink.class.getName()); + renderConfig.put("template", template); + renderConfig.put("output", Constants.INPUT_MARKER); + renderConfig.put("variables", variables); + linksConfig.put(renderName, renderConfig); + + linkConfig.remove("input"); + } + + unfoldedLinks.add(linkName); + } + + config.put("chain", unfoldedLinks); + + return config; + } + + @SuppressWarnings("unchecked") + public static ChainLink getRootLink(Map config) { + assert config.containsKey("chain"); + assert config.get("chain") instanceof List; + List links = (List) config.get("chain"); + + assert config.containsKey("parsers"); + assert config.get("parsers") instanceof Map; + Map linksConfig = (Map) config.get("parsers"); + + assert links.size() > 0; + ChainLink prevLink = null; + ChainLink linkObject = null; + for (int i = links.size() - 1; i >= 0; i--) { + assert links.get(i) instanceof String; + String linkName = (String) links.get(i); + assert linksConfig.containsKey(linkName); + assert linksConfig.get(linkName) instanceof Map; + Map linkConfig = (Map) linksConfig.get(linkName); + assert linkConfig.containsKey("class"); + assert linkConfig.get("class") instanceof String; + String className = (String) linkConfig.get("class"); + try { + linkObject = (ChainLink) ChainLink.class.getClassLoader().loadClass(className).newInstance(); + } catch (InstantiationException e) { + throw new IllegalStateException("Could not instantiate the following link: " + className); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Illegal access exception for link: " + className); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Class not found exception for link: " + className); + } + assert linkObject != null; + linkConfig.remove("class"); + linkObject.configure(linkConfig); + if (prevLink != null) { + linkObject.setNextLink(prevLink); + } + prevLink = linkObject; + } + assert linkObject != null; + return linkObject; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java new file mode 100644 index 0000000000..71e5865c43 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.type.MapType; +import org.codehaus.jackson.map.type.TypeFactory; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class JSONUtils { + + public static Map JSONToMap(String json) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + MapType type = mapper.getTypeFactory().constructMapType(Map.class, String.class, Object.class); + return mapper.readValue(json, type); + } + + public static Map JSONToMap(JSONObject json) throws IOException { + return JSONUtils.JSONToMap(json.toString()); + } + + public static JSONObject stringToJSON(String json) throws ParseException { + JSONParser parseJSON = new JSONParser(); + return (JSONObject) parseJSON.parse(json); + } + + public static JSONObject mapToJSON(Map data) { + return new JSONObject(data); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java new file mode 100644 index 0000000000..f1d5aa02ad --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StringUtils { + + /** + * Extracts (key, value) pairs from a string efficiently. + * + * @param message Message to find (key, value) pairs in. + * @param keyValueDelimiter Regular expression for the delimiter between the key and the value (for example: "="). + * @param pairDelimiters Regular expression for the delimiter between pairs (for example: " "). + * @param validKeyChars Valid characters for the keys (e.g. "a-zA-Z"). + * @return A map in which the (key, value) pairs are stored. + */ + @SuppressWarnings("unchecked") + public static Map parseKeyValuePairs(String message, String keyValueDelimiter, + String pairDelimiters, String validKeyChars) { + Map keyValues = new HashMap<>(); + StringBuilder splitRegex = new StringBuilder("(?= -999999; + return true; + } catch (Exception e) { + return false; + } + } else return number instanceof Integer; + } + + public static int toInteger(Object number) { + if (!StringUtils.isNumerical(number)) return -1; + if (number instanceof String) { + return Integer.parseInt((String) number); + } else if (number instanceof Integer) { + return (int) number; + } + return -1; + } + + public static String normalize(String key) { + String result = org.apache.commons.lang3.StringUtils.stripAccents(key); + result = result.replaceAll("[^A-Za-z_0-9]+", "_"); + result = result.replaceAll("[_]+", "_"); + result = result.replaceAll("^[_]*(.*)[_]*$", "$1"); + result = result.toLowerCase(); + return result; + } + +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java new file mode 100644 index 0000000000..a6ef08882b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainlink; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.links.fields.IdentityLink; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestChainLink { + + private ChainLink link; + + @Before + public void setUp() { + this.link = new IdentityLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test(expected = AssertionError.class) + public void testSelfLink() { + // A self-link must not occur since it will result into infinite loops + this.link.setNextLink(this.link); + } + + @Test + public void testNextLink() { + ChainLink otherLink = new IdentityLink(); + this.link.setNextLink(otherLink); + assertEquals(otherLink, this.link.getNextLink()); + assertTrue(this.link.hasNextLink()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java new file mode 100644 index 0000000000..6f9c11763a --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainlink; + +import nl.qsight.chainlink.ChainLinkIO; +import nl.qsight.common.Constants; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The ChainLinkIO is capable of transforming a single input field to one or more output fields. + */ +public class TestChainLinkIO { + + private AddExclamationMarkLink link_str; + private IncrementListLink link_list; + private MultiOutputLink link_multi; + + private class AddExclamationMarkLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + return input + "!"; + } + } + + private class IncrementListLink extends ChainLinkIO> { + + @Override + public Object parseInputField(List input) { + List newList = new ArrayList<>(); + for (int i : input) { + newList.add(i + 1); + } + return newList; + } + } + + @SuppressWarnings("unchecked") + private class MultiOutputLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + JSONObject result = new JSONObject(); + result.put("result_1", input + "?"); + result.put("result_2", input + "!"); + return result; + } + } + + @Before + public void setUp() { + this.link_str = new AddExclamationMarkLink(); + this.link_list = new IncrementListLink(); + this.link_multi = new MultiOutputLink(); + } + + @After + public void tearDown() { + this.link_str = null; + this.link_list = null; + this.link_multi = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testStringOperations() { + JSONObject input = new JSONObject(); + input.put("var1", "test1"); + input.put("var2", "test2"); + input.put(Constants.INPUT_MARKER, "test3"); + + JSONObject output = this.link_str.parse(input); + + assertTrue(output.containsKey(Constants.OUTPUT_MARKER)); + assertTrue(output.containsKey("var1")); + assertTrue(output.containsKey("var2")); + assertEquals(3, output.size()); + assertEquals("test1", output.get("var1")); + assertEquals("test2", output.get("var2")); + assertEquals("test3!", output.get(Constants.OUTPUT_MARKER)); + } + + @Test + @SuppressWarnings("unchecked") + public void testListOperations() { + JSONObject input = new JSONObject(); + List intList = new ArrayList<>(); + intList.add(1); + intList.add(2); + intList.add(3); + input.put(Constants.INPUT_MARKER, intList); + + JSONObject output = this.link_list.parse(input); + + assertTrue(output.containsKey(Constants.OUTPUT_MARKER)); + List result = ((List) output.get(Constants.OUTPUT_MARKER)); + assertEquals(1, output.size()); + assertEquals(3, result.size()); + assertTrue(result.get(0) == 2); + assertTrue(result.get(1) == 3); + assertTrue(result.get(2) == 4); + } + + @Test + @SuppressWarnings("unchecked") + public void testMultiOutput() { + JSONObject input = new JSONObject(); + input.put(Constants.INPUT_MARKER, "test"); + + JSONObject output = this.link_multi.parse(input); + + assertFalse(output.containsKey(Constants.OUTPUT_MARKER)); + assertTrue(output.containsKey("result_1")); + assertTrue(output.containsKey("result_2")); + assertEquals(2, output.size()); + assertEquals("test?", output.get("result_1")); + assertEquals("test!", output.get("result_2")); + } + + @Test(expected = IllegalStateException.class) + public void testNoInputField() { + JSONObject input = new JSONObject(); + this.link_multi.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java new file mode 100644 index 0000000000..c17ac658fb --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.chainparser; + +import nl.qsight.chainlink.ChainLink; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.json.simple.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static nl.qsight.common.Constants.ORIGINAL_STRING; +import static nl.qsight.common.Constants.TIMESTAMP; +import static org.junit.Assert.*; + +public class TestChainParser { + + private ChainParser parser; + + private class FieldRemoveLink extends ChainLink { + + String fieldToRemove = ""; + + @Override + public JSONObject parse(JSONObject input) { + input.remove(this.fieldToRemove); + return input; + } + } + + @Before + public void setUp() { + this.parser = new ChainParser(); + } + + @After + public void tearDown() { + this.parser = null; + } + + @Test + public void testRequiredFields() { + // Even without a specified link, the required fields should be set + byte[] exampleMessage = "example_message".getBytes(); + List resultSet = this.parser.parse(exampleMessage); + + // Should contain 1 message + assertEquals(1, resultSet.size()); + + // Therefore, it is possible to access the first element of the result + JSONObject state = resultSet.get(0); + + // Check whether the state contains the required "original_string" message + assertTrue("The parsed JSONObject should contain \"original_string\" field.", + state.containsKey(ORIGINAL_STRING)); + assertTrue("The parsed JSONObject should contain \"timestamp\" field.", + state.containsKey(TIMESTAMP)); + } + + @Test + public void testGetSetInitialLink() { + FieldRemoveLink link = new FieldRemoveLink(); + this.parser.setInitialLink(link); + ChainLink initialLink = this.parser.getInitialLink(); + assertEquals("The getter of the initial link should equal the item set by its setter.", link, initialLink); + } + + @Test(expected = IllegalStateException.class) + public void testNoOriginalString() { + // An exception should be thrown when a link removes the "original_string" field + FieldRemoveLink link = new FieldRemoveLink(); + link.fieldToRemove = ORIGINAL_STRING; + this.parser.setInitialLink(link); + this.parser.parse("".getBytes()); + } + + @Test(expected = IllegalStateException.class) + public void testNoTimestamp() { + // An exception should be thrown when a link removes the "timestamp" field + FieldRemoveLink link = new FieldRemoveLink(); + link.fieldToRemove = TIMESTAMP; + this.parser.setInitialLink(link); + this.parser.parse("".getBytes()); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalEncoding() { + this.parser.setEncoding("unknown"); + this.parser.parse("".getBytes()); + } + + @Test + public void testEncodingGetterAndSetter() { + this.parser.setEncoding("encoding"); + assertEquals("encoding", this.parser.getEncoding()); + } + + @Test + public void testConfiguration() { + Map link_1_config = new HashMap<>(); + Map link_2_config = new HashMap<>(); + link_1_config.put("class", "nl.qsight.links.fields.IdentityLink"); + link_2_config.put("class", "nl.qsight.links.io.SplitLink"); + Map selector = new HashMap<>(); + selector.put("0", "first"); + link_2_config.put("delimiter", "|"); + link_2_config.put("selector", selector); + Map parser_config = new HashMap<>(); + Map links_config = new HashMap<>(); + links_config.put("identity", link_1_config); + links_config.put("split", link_2_config); + parser_config.put("parsers", links_config); + List chain = new ArrayList<>(); + chain.add("identity"); + chain.add("split"); + parser_config.put("chain", chain); + + this.parser.configure(parser_config); + List output = this.parser.parse("1|2|3".getBytes()); + assertTrue(output.size() == 1); + JSONObject message = output.get(0); + assertTrue(message.containsKey("first")); + assertEquals("1", message.get("first")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java new file mode 100644 index 0000000000..e84e81d421 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.links.fields.BlacklistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestBlacklistLink { + + private BlacklistLink link; + + @Before + public void setUp() { + this.link = new BlacklistLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetFields() { + List fields = new ArrayList<>(); + fields.add("field"); + this.link.setFields(fields); + assertEquals(fields, this.link.getFields()); + } + + @Test + @SuppressWarnings("unchecked") + public void testBlacklistKeys() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + List fields = new ArrayList<>(); + fields.add("variable2"); + fields.add("variable3"); + fields.add("variable4"); + this.link.setFields(fields); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable1")); + assertEquals(1, output.size()); + } + + @Test(expected = IllegalStateException.class) + public void testNoFields() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java new file mode 100644 index 0000000000..931850fa6f --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestNormalizeFieldLink { + + private NormalizeFieldLink link; + + @Before + public void setUp() { + this.link = new NormalizeFieldLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testNormalizeKeys() { + JSONObject input = new JSONObject(); + input.put("Ĉool.keyCamelIPCase23TestI", "test1"); + + JSONObject result = this.link.parse(input); + assertTrue(result.containsKey("cool_keycamelipcase23testi")); + assertEquals("test1", result.get("cool_keycamelipcase23testi")); + assertTrue(result.size() == 1); + } + + @Test(expected = IllegalStateException.class) + @SuppressWarnings("unchecked") + public void testDuplicateKeys() { + // Test whether an exception is thrown when two fields are normalized to the same key which can lead to + // dangerous situations + JSONObject input = new JSONObject(); + input.put("Test", "test1"); + input.put("test", "test1"); + + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java new file mode 100644 index 0000000000..41f3c310de --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.links.fields.RenameLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRenameLink { + + private RenameLink link; + + @Before + public void setUp() { + this.link = new RenameLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetRenames() { + Map renames = new HashMap<>(); + renames.put("from", "to"); + this.link.setRenames(renames); + assertEquals(renames, this.link.getRenames()); + } + + @Test + @SuppressWarnings("unchecked") + public void testRename() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + Map renames = new HashMap<>(); + renames.put("variable1", "variable3"); + renames.put("variable2", "variable4"); + this.link.setRenames(renames); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable3")); + assertTrue(output.containsKey("variable4")); + assertEquals(2, output.size()); + assertEquals(output.get("variable3"), "test1"); + assertEquals(output.get("variable4"), "test2"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java new file mode 100644 index 0000000000..1c6e612110 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.links.fields.RenderLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.time.Year; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class TestRenderLink { + + private RenderLink link; + + @Before + public void setUp() { + this.link = new RenderLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetTemplate() { + String template = "test"; + this.link.setTemplate(template); + assertEquals(template, this.link.getTemplate()); + } + + @Test + public void testGetSetOutputField() { + String field = "output_field"; + this.link.setOutputField(field); + assertEquals(field, this.link.getOutputField()); + } + + @Test + @SuppressWarnings("unchecked") + public void testRenderTemplate() { + // Test whether a template is rendered correctly + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + + List variables = new ArrayList<>(); + variables.add("variable1"); + variables.add("variable2"); + this.link.setTemplate("{{variable1}} {{variable2}}."); + this.link.setOutputField("result"); + this.link.setVariables(variables); + JSONObject result = this.link.parse(input); + + assertTrue("The resulting object should have a \"result\" field.", result.containsKey("result")); + assertEquals("The template \"{{variable1}} {{variable2}}.\" should be rendered as " + + "\"test1 test2.\".", + "test1 test2.", result.get("result")); + } + + @Test(expected = IllegalStateException.class) + public void testNoOutputField() { + JSONObject input = new JSONObject(); + + this.link.setTemplate(""); + this.link.parse(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoTemplate() { + JSONObject input = new JSONObject(); + + this.link.setOutputField("result"); + this.link.parse(input); + } + + @Test + public void testCurrentYearMethod() { + JSONObject input = new JSONObject(); + + this.link.setTemplate("{{year}}"); + this.link.setOutputField("result"); + List variables = new ArrayList<>(); + variables.add("year"); + this.link.setVariables(variables); + JSONObject result = this.link.parse(input); + + assertEquals("The year variable should be rendered as the current year.", + Year.now().toString(), result.get("result")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java new file mode 100644 index 0000000000..1c8256cc5c --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.common.Constants; +import nl.qsight.links.fields.SelectLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestSelectLink { + + private SelectLink link; + + @Before + public void setUp() { + this.link = new SelectLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetField() { + String template = "test"; + this.link.setField(template); + assertEquals(template, this.link.getField()); + } + + @Test + @SuppressWarnings("unchecked") + public void testSelectLink() { + JSONObject input = new JSONObject(); + input.put("var1", "test1"); + input.put("var2", "test2"); + + this.link.setField("var1"); + JSONObject result = this.link.parse(input); + assertEquals(3, result.size()); + assertTrue(result.containsKey("var1")); + assertTrue(result.containsKey("var2")); + assertTrue(result.containsKey(Constants.INPUT_MARKER)); + assertEquals("test1", result.get("var1")); + assertEquals("test2", result.get("var2")); + assertEquals("test1", result.get(Constants.INPUT_MARKER)); + } + + @Test(expected = IllegalStateException.class) + public void testNoField() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java new file mode 100644 index 0000000000..2b0796a5b9 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTrimValueLink { + + private TrimValueLink link; + + @Before + public void setUp() { + this.link = new TrimValueLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testTrimValueFields() { + JSONObject input = new JSONObject(); + input.put("whitespace test", " test 1 "); + + JSONObject result = this.link.parse(input); + assertTrue(result.containsKey("whitespace test")); + assertEquals("test 1", result.get("whitespace test")); + assertTrue(result.size() == 1); + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java new file mode 100644 index 0000000000..39a5426d97 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.fields; + +import nl.qsight.links.fields.WhitelistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestWhitelistLink { + + private WhitelistLink link; + + @Before + public void setUp() { + this.link = new WhitelistLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetFields() { + List fields = new ArrayList<>(); + fields.add("field"); + this.link.setFields(fields); + assertEquals(fields, this.link.getFields()); + } + + @Test + @SuppressWarnings("unchecked") + public void testWhitelistKeys() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + List fields = new ArrayList<>(); + fields.add("variable2"); + fields.add("variable3"); + fields.add("variable4"); + this.link.setFields(fields); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable2")); + assertTrue(output.containsKey("variable3")); + assertEquals(2, output.size()); + } + + @Test(expected = IllegalStateException.class) + public void testNoFields() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java new file mode 100644 index 0000000000..9b43a6fac2 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import nl.qsight.common.Constants; +import nl.qsight.links.fields.WhitelistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestJSONDecodeLink { + + private JSONDecoderLink link; + + @Before + public void setUp() { + this.link = new JSONDecoderLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testJSONDecode() { + String input = "{\"a\": {\"b\": \"c\"}, \"d\": \"e\"}"; + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertEquals(2, output.size()); + assertTrue(output.containsKey("a.b")); + assertTrue(output.containsKey("d")); + assertEquals("c", output.get("a.b")); + assertEquals("e", output.get("d")); + } + + @Test(expected = IllegalStateException.class) + public void testParseError() { + String input = "non-json"; + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java new file mode 100644 index 0000000000..fd062472bf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestKeyValueLink { + + private KeyValueLink link; + + @Before + public void setUp() { + this.link = new KeyValueLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testKeyValueLink() { + String input = "hello=world|message=test"; + + this.link.setKeyValueDelimiter("="); + this.link.setPairDelimiter("|"); + this.link.setValidKeyChars("a-z"); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertEquals(2, output.size()); + assertTrue(output.containsKey("hello")); + assertTrue(output.containsKey("message")); + assertEquals("world", output.get("hello")); + assertEquals("test", output.get("message")); + } + + @Test(expected = IllegalStateException.class) + public void testNoKeyValueDelimiter() { + String input = "hello=world|message=test"; + this.link.setPairDelimiter("|"); + this.link.setValidKeyChars("a-z"); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoPairDelimiter() { + String input = "hello=world|message=test"; + this.link.setKeyValueDelimiter("="); + this.link.setValidKeyChars("a-z"); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoValidCharacters() { + String input = "hello=world|message=test"; + this.link.setKeyValueDelimiter("="); + this.link.setPairDelimiter("|"); + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java new file mode 100644 index 0000000000..64fbdef3bf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRegexLink { + + private RegexLink link; + + @Before + public void setUp() { + this.link = new RegexLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testRegexLink() { + String input = "01-01-1970"; + Map selector = new HashMap<>(); + selector.put("day", "1"); + selector.put("month", "2"); + selector.put("year", "3"); + selector.put("all", "0"); + + this.link.setSelector(selector); + this.link.setPattern("([0-9]{1,2})-([0-9]{1,2})-([0-9]{4})"); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(4, output.size()); + assertTrue(output.containsKey("all")); + assertTrue(output.containsKey("month")); + assertTrue(output.containsKey("year")); + assertTrue(output.containsKey("day")); + assertEquals("01", output.get("day")); + assertEquals("01", output.get("month")); + assertEquals("1970", output.get("year")); + assertEquals("01-01-1970", output.get("all")); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalIndex() { + String input = "test_input"; + Map selector = new HashMap<>(); + selector.put("test_key", "illegal_index"); + this.link.setSelector(selector); + this.link.setPattern("(.*)"); + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java new file mode 100644 index 0000000000..15a6bc196a --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestSplitLink { + + private SplitLink link; + + @Before + public void setUp() { + this.link = new SplitLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testSplitLink() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("-1", "test_1"); + selector.put("1", "test_2"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(2, output.size()); + assertTrue(output.containsKey("test_1")); + assertTrue(output.containsKey("test_2")); + assertEquals("some_footer", output.get("test_1")); + assertEquals("some_message", output.get("test_2")); + } + + @Test + public void testSplitLinkRegex() { + String input = "some_header||||some_message||some_footer"; + Map selector = new HashMap<>(); + selector.put("-1", "test_1"); + selector.put("1", "test_2"); + + this.link.setDelimiter("\\|+", true); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(2, output.size()); + assertTrue(output.containsKey("test_1")); + assertTrue(output.containsKey("test_2")); + assertEquals("some_footer", output.get("test_1")); + assertEquals("some_message", output.get("test_2")); + } + + @Test + public void testOutOfBounds() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("5", "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(0, output.size()); + } + + @Test + public void testIntegerPositions() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put(1, "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(1, output.size()); + assertTrue(output.containsKey("test_1")); + assertEquals("some_message", output.get("test_1")); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalIndex() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("illegal_index", "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalDelimiter() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("0", "test_1"); + this.link.setSelector(selector); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalSelector() { + String input = "some_header|some_message|some_footer"; + this.link.setDelimiter("|"); + this.link.parseInputField(input); + } + + @Test + public void testGetSetDelimiter() { + this.link.setDelimiter("|"); + assertEquals("|", this.link.getDelimiter()); + assertEquals(false, this.link.isDelimiterRegex()); + this.link.setDelimiter(".*", true); + assertEquals(".*", this.link.getDelimiter()); + assertEquals(true, this.link.isDelimiterRegex()); + } + + @Test + public void testGetSetSelector() { + Map selector = new HashMap<>(); + selector.put("0", "test_1"); + this.link.setSelector(selector); + assertEquals(selector, this.link.getSelector()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java new file mode 100644 index 0000000000..89ab129115 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTimestampLink { + + private TimestampLink link; + + @Before + public void setUp() { + this.link = new TimestampLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testTimestamp() { + String input = "1488801958"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("timestamp"); + pattern.add("newest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + this.link.initPredefinedPatterns(); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-03-06 12:05:58.000 +0000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS Z", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 1, output.get("timezone_available")); + } + + @Test(expected = IllegalStateException.class) + @java.lang.SuppressWarnings("unchecked") + public void testIllegalInput() { + String input = "9999-99-99 99:99:99"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + this.link.parseInputField(input); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testDatetimeWithoutTimezone() { + String input = "2017-01-01 01:01:01"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-01-01 01:01:01.000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 0, output.get("timezone_available")); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testSelectionCriteria() { + String input = "2017-01-01 01:01:01 | 2017-01-01 02:01:01"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-01-01 01:01:01.000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 0, output.get("timezone_available")); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testPatternsGetterAndSetter() { + List patterns = new ArrayList(); + this.link.setPatterns(patterns); + assertEquals(patterns, this.link.getPatterns()); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testPredefinedPatternsGetterAndSetter() { + Map> predefinedPatterns = new HashMap(); + this.link.setPredefinedPatterns(predefinedPatterns); + assertEquals(predefinedPatterns, this.link.getPredefinedPatterns()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java new file mode 100644 index 0000000000..eda0b1f908 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.parserconfig; + +public class TestDhcp extends TestParser { + + @Override + public String getFolder() { + return "dhcp"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java new file mode 100644 index 0000000000..a1b75ca1ab --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.parserconfig; + +import nl.qsight.chainlink.ChainLink; +import nl.qsight.chainparser.ChainParser; +import nl.qsight.utils.JSONUtils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Test; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; + +abstract class TestParser { + + private long lastTime; + private long lastParserTime; + private long totalTime; + private long totalTimeExceptFirst; + private int runs; + + public String getFolder() { + throw new NotImplementedException(); + } + + private String readFile(String fileName) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + StringBuilder sb = new StringBuilder(); + String line = br.readLine(); + + while (line != null) { + sb.append(line); + sb.append("\n"); + line = br.readLine(); + } + return sb.toString(); + } + } + + private Map getConfig() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String json = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/config.json")).getFile()); + return JSONUtils.JSONToMap(json); + } + + private String[] getInputLoglines() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String data = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/data_input")).getFile()); + return data.split("\\r?\\n"); + } + + private String[] getOutputLoglines() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String data = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/data_output")).getFile()); + return data.split("\\r?\\n"); + } + + @Test + public void testConfigure() throws IOException { + ChainParser parser = new ChainParser(); + parser.configure(this.getConfig()); + } + + @Test + public void testInputAndOutput() throws IOException { + String inputLines[] = getInputLoglines(); + String outputLines[] = getOutputLoglines(); + assertEquals("The number of input loglines (" + inputLines.length + ") should equal the number of expected output loglines (" + outputLines.length + ").", inputLines.length, outputLines.length); + } + + public void preLinkHook(ChainLink link) { + lastTime = System.nanoTime(); + System.out.println("Link: " + link.getClass().getName()); + + } + + public void postLinkHook(ChainLink link) { + long currentTime = System.nanoTime(); + long duration = currentTime - lastTime; + this.lastParserTime += duration; + System.out.println("Duration: " + Double.toString(duration / 1000000.) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + } + + @Test + public void testParser() throws IOException, NoSuchMethodException { + JSONParser parseJSON = new JSONParser(); + + ChainParser parser = new ChainParser(); + parser.configure(this.getConfig()); + + Method preLinkHook = this.getClass().getMethod("preLinkHook", ChainLink.class); + parser.setPreLinkHook(preLinkHook, this); + + Method postLinkHook = this.getClass().getMethod("postLinkHook", ChainLink.class); + parser.setPostLinkHook(postLinkHook, this); + + String inputLines[] = getInputLoglines(); + String outputLines[] = getOutputLoglines(); + + // By testInputAndOutput we are assured that inputLines.length == outputLines.length + runs = 0; + totalTime = 0; + totalTimeExceptFirst = 0; + + // Make sure that there are at least some amount of runs + int numEpochs = Math.max(1, (int) Math.ceil(((10. + 1.) / inputLines.length))); + + for (int epoch = 0; epoch < numEpochs; epoch++) { + for (int line = 0; line < inputLines.length; line++) { + runs += 1; + String input = inputLines[line]; + String strExpectedOutput = outputLines[line]; + System.out.println("==================================================================================="); + System.out.println("Start ChainParser:"); + System.out.println("Epoch: " + Integer.toString(epoch)); + System.out.println("Logline: " + Integer.toString(line)); + System.out.println("Input: " + input); + System.out.println("Expected output: " + strExpectedOutput); + System.out.println("==================================================================================="); + JSONObject expectedOutput; + + lastParserTime = 0; + + try { + expectedOutput = (JSONObject) parseJSON.parse(strExpectedOutput); + } catch (ParseException exception) { + exception.printStackTrace(); + throw new IllegalStateException("Line " + line + " of the output file is not a valid JSON file."); + } + + List outputMessages = parser.parse(input.getBytes()); + assertEquals("There should be exactly one message produced by the parsers.", 1, outputMessages.size()); + JSONObject output = outputMessages.get(0); + assertEquals("The output by applying the parser on line " + line + " of the data_input file does not match the expected output of line " + line + " in the data_output file.", expectedOutput.toString(), output.toString()); + + totalTime += lastParserTime; + if (line > 0) totalTimeExceptFirst += lastParserTime; + System.out.println("Parser duration: " + Double.toString(lastParserTime / 1000000.) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + System.out.println(); + } + } + + System.out.println("==================================================================================="); + System.out.println("Overall statistics"); + System.out.println("==================================================================================="); + System.out.println("No. runs: " + Integer.toString(runs)); + System.out.println("Total duration: " + Double.toString(totalTime / 1000000.) + " ms"); + System.out.println("Avg. duration: " + Double.toString(totalTime / 1000000. / (double) runs) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + System.out.println(); + + System.out.println("==================================================================================="); + System.out.println("Statistics (excluding first run since it initializes all classes)"); + System.out.println("==================================================================================="); + double avgDuration = totalTimeExceptFirst / 1000000. / ((double) runs - 1); + int eps = (int) (1000. / avgDuration); + System.out.println("No. runs: " + Integer.toString(runs - 1)); + System.out.println("Total duration: " + Double.toString(totalTimeExceptFirst / 1000000.) + " ms"); + System.out.println("Avg. duration: " + Double.toString(avgDuration) + " ms"); + System.out.println("Single-node EPS: " + Integer.toString(eps)); + System.out.println("-----------------------------------------------------------------------------------"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java new file mode 100644 index 0000000000..1646fa5ec3 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.parserconfig; + +public class TestSuricata extends TestParser { + + @Override + public String getFolder() { + return "suricata"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java new file mode 100644 index 0000000000..106f58b6ce --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.parserconfig; + +public class TestYaf extends TestParser { + + @Override + public String getFolder() { + return "yaf"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java new file mode 100644 index 0000000000..0a463b03cf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import nl.qsight.links.fields.RenderLink; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static nl.qsight.common.Constants.INPUT_MARKER; +import static org.junit.Assert.assertEquals; + +public class TestConfigUtils { + + @Test + public void testUnfoldInput() throws ParseException, IOException { + String strConfig = "{\"chain\": [\"example\"], \"parsers\": {\"example\": {\"input\": \"{{var1}}\"}}}"; + JSONParser parseJSON = new JSONParser(); + JSONObject configJSON = (JSONObject) parseJSON.parse(strConfig); + Map config = JSONUtils.JSONToMap(configJSON); + Map parserConfig = ConfigUtils.unfoldInput(config); + + assertTrue(parserConfig.containsKey("chain")); + assertTrue(parserConfig.get("chain") instanceof List); + List chain = (List) parserConfig.get("chain"); + assertTrue(chain.size() == 2); + assertTrue(parserConfig.get("parsers") instanceof Map); + Map linksConfig = (Map) parserConfig.get("parsers"); + + assertTrue(chain.get(0) instanceof String); + String autolink = (String) chain.get(0); + assertTrue(linksConfig.containsKey(autolink)); + assertTrue(linksConfig.get(autolink) instanceof Map); + Map configAutolink = (Map) linksConfig.get(autolink); + + assertTrue(configAutolink.containsKey("class")); + assertTrue(configAutolink.containsKey("template")); + assertTrue(configAutolink.containsKey("output")); + + assertTrue(configAutolink.get("class") instanceof String); + assertTrue(configAutolink.get("template") instanceof String); + assertTrue(configAutolink.get("output") instanceof String); + + assertEquals(RenderLink.class.getName(), configAutolink.get("class")); + assertEquals("{{var1}}", configAutolink.get("template")); + assertEquals(INPUT_MARKER, configAutolink.get("output")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java new file mode 100644 index 0000000000..18c1ca188b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import org.json.simple.JSONObject; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +public class TestJSONUtils { + + @Test + public void testJSONStringToMap() throws IOException { + String json = "{\"a\": \"b\"}"; + Map result = JSONUtils.JSONToMap(json); + assertTrue(result.containsKey("a")); + assertEquals(1, result.size()); + assertTrue(result.get("a") instanceof String); + assertEquals("b", result.get("a")); + } + + @Test + public void testJSONObjectToMap() throws ParseException, IOException { + String json = "{\"a\": \"b\"}"; + JSONObject jsonObject = JSONUtils.stringToJSON(json); + Map result = JSONUtils.JSONToMap(jsonObject); + assertTrue(result.containsKey("a")); + assertEquals(1, result.size()); + assertTrue(result.get("a") instanceof String); + assertEquals("b", result.get("a")); + } + + @Test + public void testStringToJSON() throws ParseException { + String json = "{\"a\": \"b\"}"; + JSONObject jsonObject = JSONUtils.stringToJSON(json); + assertTrue(jsonObject.containsKey("a")); + assertTrue(jsonObject.get("a") instanceof String); + assertEquals(jsonObject.get("a"), "b"); + } + + @Test + public void testMapToJSON() throws IOException { + String json = "{\"a\": \"b\"}"; + Map map = JSONUtils.JSONToMap(json); + JSONObject jsonObject = JSONUtils.mapToJSON(map); + assertTrue(jsonObject.containsKey("a")); + assertTrue(jsonObject.get("a") instanceof String); + assertEquals(jsonObject.get("a"), "b"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java new file mode 100644 index 0000000000..4e1c842404 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.qsight.utils; + +import org.junit.Test; + +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +// @todo add tests + +public class TestStringUtils { + + @Test + public void testParseKeyValuePairs() { + Map pairs = StringUtils.parseKeyValuePairs("hello: world, test: message", ": ", ",", "a-z"); + + assertEquals(2, pairs.size()); + assertTrue(pairs.containsKey("test")); + assertTrue(pairs.containsKey("hello")); + assertEquals("world", pairs.get("hello")); + assertEquals("message", pairs.get("test")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json new file mode 100644 index 0000000000..57395684e7 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json @@ -0,0 +1,72 @@ +{ + "chain": [ + "key_value_parser", + "trim_whitespaces", + "parse_datetime", + "parse_macaddress", + "parse_message_type", + "parse_hostname", + "normalize_fields", + "rename_fields" + ], + "parsers": { + "key_value_parser": { + "class": "nl.qsight.links.io.KeyValueLink", + "pair_delimiter": "|", + "key_value_delimiter": ":", + "valid_key_characters": "a-zA-z" + }, + "trim_whitespaces": { + "class" : "nl.qsight.links.fields.TrimValueLink" + }, + "parse_datetime": { + "class": "nl.qsight.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", + "yyyy MM dd HH mm ss SSS", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss SSS Z", + "newest" + ] + ], + "input": "{{TIME}}" + }, + "parse_macaddress": { + "class": "nl.qsight.links.io.RegexLink", + "pattern": "(Client-identifier: )([^\\|]++)", + "selector": { + "mac_address": "2" + }, + "input": "{{original_string}}" + }, + "parse_message_type": { + "class": "nl.qsight.links.io.RegexLink", + "pattern": "(DHCP message type: \\d \\|)([\\w]++)", + "selector": { + "message_type": "2" + }, + "input": "{{original_string}}" + }, + "parse_hostname": { + "class": "nl.qsight.links.io.RegexLink", + "pattern": "(Host name: )([^\\|]++)", + "selector": { + "hostname": "2" + }, + "input": "{{original_string}}" + }, + "normalize_fields": { + "class" : "nl.qsight.links.fields.NormalizeFieldLink" + }, + "rename_fields": { + "class": "nl.qsight.links.fields.RenameLink", + "rename": { + "ciaddr": "ip_src_addr" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input new file mode 100644 index 0000000000..15f311bc84 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input @@ -0,0 +1 @@ +{TIME: 2017-01-16 16:36:17.249|INTERFACE: eth2|OP:1 BOOTPREQUEST|CIADDR: 111.111.111.111|YIADDR: 0.0.0.0|SIADDR: 0.0.0.0|GIADDR: 111.111.111.111|CHADDR: 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00|OPTION: 53 1 DHCP message type: 8 |DHCPINFORM|OPTION: 61 7 Client-identifier: 00:00:00:00:00:00:00|OPTION: 12 5 Host name: A1234|OPTION: 60 8 Vendor class identifier: MSFT 5.0|OPTION: 55 13 Parameter Request List: 1 (Subnet mask)|| 15 (Domainname)|| 3 (Routers)|| 6 (DNS server)|| 44 (NetBIOS name server)|| 46 (NetBIOS node type)|| 47 (NetBIOS scope)|| 31 (Perform router discovery)|| 33 (Static route)||121 (Classless Static Route)||249 (MSFT - Classless route)|| 43 (Vendor specific info)||252 (MSFT - WinSock Proxy Auto Detect)|||IP: 111.111.111.111 > 111.111.111.111 | 00:00:00:00:00:00 > 00:00:00:00:00:00} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output new file mode 100644 index 0000000000..55ea64e949 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output @@ -0,0 +1 @@ +{"chaddr":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","op":"1 BOOTPREQUEST","mapping":"yyyy-MM-dd HH:mm:ss.SSS","timezone_available":0,"ip":"111.111.111.111 > 111.111.111.111","message_type":"DHCPINFORM","giaddr":"111.111.111.111","interface":"eth2","yiaddr":"0.0.0.0","datetime":"2017-01-16 16:36:17.249","hostname":"A1234","original_string":"{TIME: 2017-01-16 16:36:17.249|INTERFACE: eth2|OP:1 BOOTPREQUEST|CIADDR: 111.111.111.111|YIADDR: 0.0.0.0|SIADDR: 0.0.0.0|GIADDR: 111.111.111.111|CHADDR: 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00|OPTION: 53 1 DHCP message type: 8 |DHCPINFORM|OPTION: 61 7 Client-identifier: 00:00:00:00:00:00:00|OPTION: 12 5 Host name: A1234|OPTION: 60 8 Vendor class identifier: MSFT 5.0|OPTION: 55 13 Parameter Request List: 1 (Subnet mask)|| 15 (Domainname)|| 3 (Routers)|| 6 (DNS server)|| 44 (NetBIOS name server)|| 46 (NetBIOS node type)|| 47 (NetBIOS scope)|| 31 (Perform router discovery)|| 33 (Static route)||121 (Classless Static Route)||249 (MSFT - Classless route)|| 43 (Vendor specific info)||252 (MSFT - WinSock Proxy Auto Detect)|||IP: 111.111.111.111 > 111.111.111.111 | 00:00:00:00:00:00 > 00:00:00:00:00:00}","original_timestamp":"2017-01-16 16:36:17.249","siaddr":"0.0.0.0","mac_address":"00:00:00:00:00:00:00","time":"2017-01-16 16:36:17.249","ip_src_addr":"111.111.111.111","option":"55 13 Parameter Request List: 1 (Subnet mask)","timestamp":1484584577249} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json new file mode 100644 index 0000000000..633a3f5e28 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json @@ -0,0 +1,42 @@ +{ + "chain": [ + "parse_json", + "parse_username", + "rename_fields", + "parse_datetime" + ], + "parsers": { + "parse_json": { + "class": "nl.qsight.links.io.JSONDecoderLink" + }, + "parse_username": { + "class": "nl.qsight.links.io.RegexLink", + "pattern": "(?i)(user|username|log)[=:](\\w+)", + "selector": { + "username": "2" + }, + "input": "{{payload_printable}}" + }, + "rename_fields": { + "class": "nl.qsight.links.fields.RenameLink", + "rename": { + "proto": "protocol", + "dest_ip": "ip_dst_addr", + "src_ip": "ip_src_addr", + "dest_port": "ip_dst_port", + "src_port": "ip_src_port" + } + }, + "parse_datetime": { + "class": "nl.qsight.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", + "yyyy MM dd HH mm ss SSSSSS Z", + "newest" + ] + ], + "input": "{{timestamp}}" + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input new file mode 100644 index 0000000000..731679f19e --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input @@ -0,0 +1,3 @@ +{"timestamp":"2017-01-11T00:54:01.918498+0100","flow_id":111,"in_iface":"eth0","event_type":"dns","src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":1234,"proto":"UDP","dns":{"type":"query","id":1111,"rrname":"x.local","rrtype":"A","tx_id":0}} +{"timestamp":"2017-09-07T15:10:30.292276+0200","flow_id":111,"in_iface":"eth0","event_type":"dns","vlan":1,"src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":1234,"proto":"UDP","dns":{"type":"answer","id":111,"rcode":"NOERROR","rrname":"x.local","rrtype":"NS","ttl":3600,"rdata":"y.local"}} +{"timestamp":"2017-10-23T14:15:58.514793+0200","flow_id":111,"in_iface":"eth0","event_type":"alert","vlan":1,"src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":1234,"rev":3,"signature":"ET POLICY Http Client Body contains pwd= in cleartext","category":"Potential Corporate Privacy Violation","severity":1},"http":{"hostname":"x.com","url":"\/login.php?op=cusLogin","http_user_agent":"Apache-HttpClient\/UNAVAILABLE (java 1.4)","http_content_type":"application\/json","http_method":"POST","protocol":"HTTP\/1.1","status":200,"length":60},"payload":"nopayload","payload_printable":"pwd=1234&userName=user1","stream":1,"packet":"1234","packet_info":{"linktype":1}} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output new file mode 100644 index 0000000000..9bb69dbc84 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output @@ -0,0 +1,3 @@ +{"dns.type":"query","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","ip_dst_port":1234,"timezone_available":1,"dns.id":1111,"in_iface":"eth0","dns.rrname":"x.local","protocol":"UDP","datetime":"2017-01-11 00:54:01.918 +0100","original_string":"{\"timestamp\":\"2017-01-11T00:54:01.918498+0100\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"dns\",\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":1234,\"proto\":\"UDP\",\"dns\":{\"type\":\"query\",\"id\":1111,\"rrname\":\"x.local\",\"rrtype\":\"A\",\"tx_id\":0}}","event_type":"dns","dns.tx_id":0,"ip_dst_addr":"111.111.111.111","original_timestamp":"2017-01-11 00:54:01.918","ip_src_port":1234,"flow_id":111,"dns.rrtype":"A","ip_src_addr":"111.111.111.111","timestamp":1484092441918} +{"dns.type":"answer","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","ip_dst_port":1234,"timezone_available":1,"dns.rcode":"NOERROR","dns.id":111,"dns.ttl":3600,"dns.rdata":"y.local","in_iface":"eth0","dns.rrname":"x.local","protocol":"UDP","datetime":"2017-09-07 15:10:30.292 +0200","original_string":"{\"timestamp\":\"2017-09-07T15:10:30.292276+0200\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"dns\",\"vlan\":1,\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":1234,\"proto\":\"UDP\",\"dns\":{\"type\":\"answer\",\"id\":111,\"rcode\":\"NOERROR\",\"rrname\":\"x.local\",\"rrtype\":\"NS\",\"ttl\":3600,\"rdata\":\"y.local\"}}","event_type":"dns","ip_dst_addr":"111.111.111.111","original_timestamp":"2017-09-07 15:10:30.292","vlan":1,"ip_src_port":1234,"flow_id":111,"dns.rrtype":"NS","ip_src_addr":"111.111.111.111","timestamp":1504789830292} +{"alert.category":"Potential Corporate Privacy Violation","ip_dst_port":80,"http.http_method":"POST","timezone_available":1,"http.url":"\/login.php?op=cusLogin","http.http_content_type":"application\/json","protocol":"TCP","datetime":"2017-10-23 14:15:58.514 +0200","original_string":"{\"timestamp\":\"2017-10-23T14:15:58.514793+0200\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"alert\",\"vlan\":1,\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":80,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":1234,\"rev\":3,\"signature\":\"ET POLICY Http Client Body contains pwd= in cleartext\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1},\"http\":{\"hostname\":\"x.com\",\"url\":\"\\\/login.php?op=cusLogin\",\"http_user_agent\":\"Apache-HttpClient\\\/UNAVAILABLE (java 1.4)\",\"http_content_type\":\"application\\\/json\",\"http_method\":\"POST\",\"protocol\":\"HTTP\\\/1.1\",\"status\":200,\"length\":60},\"payload\":\"nopayload\",\"payload_printable\":\"pwd=1234&userName=user1\",\"stream\":1,\"packet\":\"1234\",\"packet_info\":{\"linktype\":1}}","event_type":"alert","ip_dst_addr":"111.111.111.111","vlan":1,"alert.rev":3,"payload":"nopayload","stream":1,"flow_id":111,"alert.signature_id":1234,"alert.action":"allowed","packet_info.linktype":1,"ip_src_addr":"111.111.111.111","timestamp":1508760958514,"alert.severity":1,"mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","payload_printable":"pwd=1234&userName=user1","tx_id":0,"packet":"1234","http.protocol":"HTTP\/1.1","http.length":60,"in_iface":"eth0","http.hostname":"x.com","alert.gid":1,"original_timestamp":"2017-10-23 14:15:58.514","alert.signature":"ET POLICY Http Client Body contains pwd= in cleartext","ip_src_port":1234,"http.status":200,"http.http_user_agent":"Apache-HttpClient\/UNAVAILABLE (java 1.4)","username":"user1"} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json new file mode 100644 index 0000000000..9799e4e727 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json @@ -0,0 +1,69 @@ +{ + "chain": [ + "values_splitter", + "trim_whitespaces", + "parse_datetime", + "rename_fields" + ], + "parsers": { + "values_splitter": { + "class": "nl.qsight.links.io.SplitLink", + "delimiter": "|", + "selector": { + "1": "start-time", + "2": "end-time", + "3": "duration", + "4": "rtt", + "5": "proto", + "6": "sip", + "7": "sp", + "8": "dip", + "9": "dp", + "10": "srcMacAddress", + "11": "destMacAddress", + "12": "iflags", + "13": "uflags", + "14": "riflags", + "15": "ruflags", + "16": "isn", + "17": "risn", + "18": "tag", + "19": "rtag", + "20": "pkt", + "21": "oct", + "22": "rpkt", + "23": "roct", + "24": "app", + "25": "end-reason" + } + }, + "parse_datetime": { + "class": "nl.qsight.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", + "yyyy MM dd HH mm ss SSS", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss SSS Z", + "newest" + ] + ], + "input": "{{end-time}}" + }, + "trim_whitespaces": { + "class" : "nl.qsight.links.fields.TrimValueLink" + }, + "rename_fields": { + "class": "nl.qsight.links.fields.RenameLink", + "rename": { + "sip": "ip_src_addr", + "sp": "ip_src_port", + "dip": "ip_dst_addr", + "dp": "ip_dst_port" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input new file mode 100644 index 0000000000..62a198eee0 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input @@ -0,0 +1 @@ +{2016-12-14 09:36:31.181|2016-12-14 09:36:31.442| 0.261| 0.000| 6| 111.111.111.111|11210| 111.111.111.111| 443|00:00:00:00:00:00|00:00:00:00:00:00| S| APF| AS| APF|aaa|aaa|000|000| 19| 4811| 16| 7059| 0|} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output new file mode 100644 index 0000000000..daba543e7a --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output @@ -0,0 +1 @@ +{"iflags":"APF","uflags":"AS","ip_dst_port":"00:00:00:00:00:00","isn":"aaa","timezone_available":0,"destMacAddress":"S","duration":"0.000","rpkt":"7059","datetime":"2016-12-14 09:36:31.442","original_string":"{2016-12-14 09:36:31.181|2016-12-14 09:36:31.442| 0.261| 0.000| 6| 111.111.111.111|11210| 111.111.111.111| 443|00:00:00:00:00:00|00:00:00:00:00:00| S| APF| AS| APF|aaa|aaa|000|000| 19| 4811| 16| 7059| 0|}","ip_dst_addr":"443","pkt":"4811","ruflags":"aaa","tag":"000","roct":"0","rtag":"19","ip_src_addr":"11210","timestamp":1481708191442,"app":"}","oct":"16","mapping":"yyyy-MM-dd HH:mm:ss.SSS","start-time":"2016-12-14 09:36:31.442","end-time":"0.261","risn":"000","rtt":"6","riflags":"APF","original_timestamp":"2016-12-14 09:36:31.442","ip_src_port":"111.111.111.111","proto":"111.111.111.111","srcMacAddress":"00:00:00:00:00:00"} \ No newline at end of file diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index 3899e9def7..708a526464 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -46,6 +46,7 @@ metron-enrichment metron-solr metron-parsers + metron-parsers-contrib metron-pcap-backend metron-data-management metron-pcap diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java index 3528737133..9915837fc1 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java index df661a9159..267d5936eb 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java index 718a4fe565..66d273a196 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java index 3bd4ad161f..21c54153d0 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF From 381b4d37dd781b4700cec2d3396d27bedcd4ac10 Mon Sep 17 00:00:00 2001 From: Kevin Jacobs Date: Wed, 21 Mar 2018 09:34:15 +0100 Subject: [PATCH 2/5] - Fix namespace. --- .../metron-parsers-contrib/README.md | 40 +++++++++---------- .../metron-parsers-contrib/pom.xml | 2 +- .../parsers/contrib}/chainlink/ChainLink.java | 5 ++- .../contrib}/chainlink/ChainLinkIO.java | 4 +- .../contrib}/chainparser/ChainParser.java | 15 +++---- .../parsers/contrib}/common/Constants.java | 2 +- .../contrib}/links/fields/BlacklistLink.java | 5 ++- .../contrib}/links/fields/IdentityLink.java | 4 +- .../links/fields/NormalizeFieldLink.java | 8 ++-- .../contrib}/links/fields/RenameLink.java | 4 +- .../contrib}/links/fields/RenderLink.java | 4 +- .../contrib}/links/fields/SelectLink.java | 6 +-- .../contrib}/links/fields/TrimValueLink.java | 6 +-- .../contrib}/links/fields/WhitelistLink.java | 4 +- .../contrib}/links/io/JSONDecoderLink.java | 5 ++- .../contrib}/links/io/KeyValueLink.java | 8 ++-- .../parsers/contrib}/links/io/RegexLink.java | 8 ++-- .../parsers/contrib}/links/io/SplitLink.java | 8 ++-- .../contrib}/links/io/TimestampLink.java | 10 +++-- .../parsers/contrib}/utils/ConfigUtils.java | 15 ++++--- .../parsers/contrib}/utils/JSONUtils.java | 2 +- .../parsers/contrib}/utils/StringUtils.java | 2 +- .../contrib}/chainlink/TestChainLink.java | 5 +-- .../contrib}/chainlink/TestChainLinkIO.java | 7 ++-- .../contrib}/chainparser/TestChainParser.java | 22 +++++----- .../links/fields/TestBlacklistLink.java | 4 +- .../links/fields/TestNormalizeFieldLink.java | 2 +- .../contrib}/links/fields/TestRenameLink.java | 4 +- .../contrib}/links/fields/TestRenderLink.java | 4 +- .../contrib}/links/fields/TestSelectLink.java | 7 ++-- .../links/fields/TestTrimValueLink.java | 2 +- .../links/fields/TestWhitelistLink.java | 4 +- .../contrib}/links/io/TestJSONDecodeLink.java | 6 +-- .../contrib}/links/io/TestKeyValueLink.java | 2 +- .../contrib}/links/io/TestRegexLink.java | 2 +- .../contrib}/links/io/TestSplitLink.java | 2 +- .../contrib}/links/io/TestTimestampLink.java | 2 +- .../contrib}/parserconfig/TestDhcp.java | 2 +- .../contrib}/parserconfig/TestParser.java | 13 +++--- .../contrib}/parserconfig/TestSuricata.java | 2 +- .../contrib}/parserconfig/TestYaf.java | 2 +- .../contrib}/utils/TestConfigUtils.java | 11 +++-- .../parsers/contrib}/utils/TestJSONUtils.java | 2 +- .../contrib}/utils/TestStringUtils.java | 2 +- .../src/test/resources/dhcp/config.json | 16 ++++---- .../src/test/resources/suricata/config.json | 8 ++-- .../src/test/resources/yaf/config.json | 8 ++-- 47 files changed, 167 insertions(+), 141 deletions(-) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainlink/ChainLink.java (92%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainlink/ChainLinkIO.java (96%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainparser/ChainParser.java (93%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/common/Constants.java (95%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/BlacklistLink.java (93%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/IdentityLink.java (91%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/NormalizeFieldLink.java (85%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/RenameLink.java (95%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/RenderLink.java (97%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/SelectLink.java (92%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TrimValueLink.java (90%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/WhitelistLink.java (95%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/JSONDecoderLink.java (92%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/KeyValueLink.java (91%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/RegexLink.java (90%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/SplitLink.java (92%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TimestampLink.java (97%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/ConfigUtils.java (91%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/JSONUtils.java (97%) rename metron-platform/metron-parsers-contrib/src/main/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/StringUtils.java (98%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainlink/TestChainLink.java (93%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainlink/TestChainLinkIO.java (95%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/chainparser/TestChainParser.java (85%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestBlacklistLink.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestNormalizeFieldLink.java (97%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestRenameLink.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestRenderLink.java (96%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestSelectLink.java (90%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestTrimValueLink.java (96%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/fields/TestWhitelistLink.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TestJSONDecodeLink.java (91%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TestKeyValueLink.java (98%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TestRegexLink.java (98%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TestSplitLink.java (99%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/links/io/TestTimestampLink.java (99%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/parserconfig/TestDhcp.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/parserconfig/TestParser.java (95%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/parserconfig/TestSuricata.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/parserconfig/TestYaf.java (94%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/TestConfigUtils.java (86%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/TestJSONUtils.java (98%) rename metron-platform/metron-parsers-contrib/src/test/java/{nl/qsight => org/apache/metron/parsers/contrib}/utils/TestStringUtils.java (96%) diff --git a/metron-platform/metron-parsers-contrib/README.md b/metron-platform/metron-parsers-contrib/README.md index 35e446a6dd..d0433ea50b 100644 --- a/metron-platform/metron-parsers-contrib/README.md +++ b/metron-platform/metron-parsers-contrib/README.md @@ -31,7 +31,7 @@ The identity link passes on the given JSON object. No changes are made to the in "chain": ["identity"], "parsers": { "identity": { - "class": "nl.qsight.links.fields.IdentityLink" + "class": "org.apache.metron.parsers.contrib.links.fields.IdentityLink" } } } @@ -46,7 +46,7 @@ The normalize field link applies the normalize function to all fields. The field "chain": ["normalize_fields"], "parsers": { "normalize_fields": { - "class": "nl.qsight.links.fields.NormalizeFieldLink" + "class": "org.apache.metron.parsers.contrib.links.fields.NormalizeFieldLink" } } } @@ -61,7 +61,7 @@ The rename link renames field names. The following configuration renames "field1 "chain": ["rename_fields"], "parsers": { "rename_fields": { - "class": "nl.qsight.links.fields.RenameLink", + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", "renames": { "field1": "fieldX", "field2": "fieldY" @@ -84,7 +84,7 @@ The following example substitutes variables "var1" and "var2" in the given templ "chain": ["render"], "parsers": { "render": { - "class": "nl.qsight.links.fields.RenderLink", + "class": "org.apache.metron.parsers.contrib.links.fields.RenderLink", "template": "Hello {{var1}} and {{var2}}", "variables": ["var1", "var2"], "output": "rendered_field" @@ -104,7 +104,7 @@ The following example selects "field1" and stores its value into the special inp "chain": ["select"], "parsers": { "select": { - "class": "nl.qsight.links.fields.SelectLink", + "class": "org.apache.metron.parsers.contrib.links.fields.SelectLink", "field": "field1" } } @@ -120,7 +120,7 @@ The trim value link trims whitespace for each value in the intermediate JSON obj "chain": ["trim_values"], "parsers": { "trim_values": { - "class": "nl.qsight.links.fields.TrimValueLink" + "class": "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" } } } @@ -137,7 +137,7 @@ The following example removes "field1" and "field2" from the input. The fields a "chain": ["blacklist"], "parsers": { "blacklist": { - "class": "nl.qsight.links.fields.BlacklistLink", + "class": "org.apache.metron.parsers.contrib.links.fields.BlacklistLink", "fields": ["field1", "field2"] } } @@ -155,7 +155,7 @@ The following example whitelists "field1" and "field2" so these are the only fie "chain": ["whitelist"], "parsers": { "trim_values": { - "class": "nl.qsight.links.fields.WhitelistLink", + "class": "org.apache.metron.parsers.contrib.links.fields.WhitelistLink", "fields": ["field1", "field2"] } } @@ -177,7 +177,7 @@ The JSON Decoder link decodes JSON found in the input field. "chain": ["json_decoder"], "parsers": { "json_decoder": { - "class": "nl.qsight.links.io.JSONDecoderLink" + "class": "org.apache.metron.parsers.contrib.links.io.JSONDecoderLink" } } } @@ -196,7 +196,7 @@ The pair delimiter would be `|` and the key-value delimiter is `=`. The followin "chain": ["keyvalue"], "parsers": { "keyvalue": { - "class": "nl.qsight.links.io.KeyValueLink", + "class": "org.apache.metron.parsers.contrib.links.io.KeyValueLink", "pair_delimiter": "|", "key_value_delimiter": "=", "valid_key_characters": "A-Z" @@ -218,7 +218,7 @@ Take a look at the following example: "chain": ["regex"], "parsers": { "regex": { - "class": "nl.qsight.links.io.RegexLink", + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", "pattern": "(?i)(user|username)[=:](\\w+)", "selector": { "username": "2" @@ -245,7 +245,7 @@ Take a look at the following example: "chain": ["split"], "parsers": { "split": { - "class": "nl.qsight.links.io.SplitLink", + "class": "org.apache.metron.parsers.contrib.links.io.SplitLink", "delimiter": "|", "selector": { "-1": "last_field", @@ -268,7 +268,7 @@ The timestamp link uses regular expressions to search for datetime patterns in t "chain": ["parse_datetime"], "parsers": { "parse_datetime": { - "class": "nl.qsight.links.io.TimestampLink", + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", "patterns": [ ["([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", "yyyy MM dd HH mm ss SSSSSS Z", "newest"] ] @@ -301,7 +301,7 @@ The timestamp parser generates the following fields: A ChainParser mainly consists of configuration and uses the ChainParser base class. The configuration should be created under `test/resources/your_parser_name/config.json`. Take a look at different ChainParser configuration files for inspiration. Then, a test file need to be created (`test/java/nl/qsight/parserconfig/TestYourParserName`). It should contain the following contents: ```java -package nl.qsight.parserconfig; +package org.apache.metron.parsers.contrib.parserconfig; public class TestYourParserName extends TestParser { @@ -326,22 +326,22 @@ Logline: 0 Input: ... Expected output: {...} =================================================================================== -Link: nl.qsight.links.io.JSONDecoderLink +Link: org.apache.metron.parsers.contrib.links.io.JSONDecoderLink Duration: 12.724098 ms ----------------------------------------------------------------------------------- -Link: nl.qsight.links.fields.RenderLink +Link: org.apache.metron.parsers.contrib.links.fields.RenderLink Duration: 0.043077 ms ----------------------------------------------------------------------------------- -Link: nl.qsight.links.io.RegexLink +Link: org.apache.metron.parsers.contrib.links.io.RegexLink Duration: 1.275897 ms ----------------------------------------------------------------------------------- -Link: nl.qsight.links.fields.RenameLink +Link: org.apache.metron.parsers.contrib.links.fields.RenameLink Duration: 0.041026 ms ----------------------------------------------------------------------------------- -Link: nl.qsight.links.fields.RenderLink +Link: org.apache.metron.parsers.contrib.links.fields.RenderLink Duration: 0.03159 ms ----------------------------------------------------------------------------------- -Link: nl.qsight.links.io.TimestampLink +Link: org.apache.metron.parsers.contrib.links.io.TimestampLink Duration: 2.045538 ms ----------------------------------------------------------------------------------- Parser duration: 16.161226 ms diff --git a/metron-platform/metron-parsers-contrib/pom.xml b/metron-platform/metron-parsers-contrib/pom.xml index e016588c2d..c696fc4927 100644 --- a/metron-platform/metron-parsers-contrib/pom.xml +++ b/metron-platform/metron-parsers-contrib/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - nl.qsight + org.apache.metron.parsers.contrib metron-parsers-contrib 1.0.0 diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java similarity index 92% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java index 9a202d84da..d2911a9a14 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainlink; +package org.apache.metron.parsers.contrib.chainlink; -import nl.qsight.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; import org.json.simple.JSONObject; import java.io.Serializable; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java similarity index 96% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java index f8bc727016..54f93db650 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainlink/ChainLinkIO.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainlink; +package org.apache.metron.parsers.contrib.chainlink; -import nl.qsight.common.Constants; +import org.apache.metron.parsers.contrib.common.Constants; import org.json.simple.JSONObject; /** diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java similarity index 93% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java index 57f458aecf..e7b5afc6a5 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/chainparser/ChainParser.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainparser; +package org.apache.metron.parsers.contrib.chainparser; -import nl.qsight.chainlink.ChainLink; -import nl.qsight.common.Constants; -import nl.qsight.links.fields.IdentityLink; -import nl.qsight.utils.ConfigUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; +import org.apache.metron.parsers.contrib.utils.ConfigUtils; import org.apache.metron.parsers.BasicParser; +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; import org.json.simple.JSONObject; import java.io.UnsupportedEncodingException; @@ -32,8 +33,8 @@ import java.util.List; import java.util.Map; -import static nl.qsight.common.Constants.ORIGINAL_STRING; -import static nl.qsight.common.Constants.TIMESTAMP; +import static org.apache.metron.parsers.contrib.common.Constants.ORIGINAL_STRING; +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; /** * The ChainParser is a composable unit consisting of a DAG of ChainLink objects. It points to the first ChainLink of diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java similarity index 95% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java index 8d2d9f2f3b..187b9ac3fc 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/common/Constants.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.common; +package org.apache.metron.parsers.contrib.common; public class Constants { diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java similarity index 93% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java index 2478e1d4c3..50de6618d5 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/BlacklistLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; import org.json.simple.JSONObject; import java.util.List; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java similarity index 91% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java index 1f3de8475b..538a1a3908 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/IdentityLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; import org.json.simple.JSONObject; /** diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java similarity index 85% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java index 8b2eb6e6ce..b259587bcd 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/NormalizeFieldLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; -import nl.qsight.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; import org.json.simple.JSONObject; /** diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java similarity index 95% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java index c4d0759618..6bbb2972c5 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenameLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; import org.json.simple.JSONObject; import java.util.Map; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java similarity index 97% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java index 26ace1aa34..cc2b8d3ad1 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/RenderLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java @@ -15,13 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; import com.google.common.collect.Maps; import com.hubspot.jinjava.Jinjava; import com.hubspot.jinjava.interpret.RenderResult; import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; import org.json.simple.JSONObject; import java.io.Serializable; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java similarity index 92% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java index 4990a2f487..2e1eb2b0b0 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/SelectLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; -import nl.qsight.common.Constants; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; import org.json.simple.JSONObject; import java.util.Map; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java similarity index 90% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java index 56ae7157fa..c3e580fd7c 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/TrimValueLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; -import nl.qsight.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; import org.json.simple.JSONObject; /** diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java similarity index 95% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java index d78d23193d..c6e58a14be 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/fields/WhitelistLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; import org.json.simple.JSONObject; import java.util.List; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java similarity index 92% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java index 9cfac8990f..e7b439b624 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/JSONDecoderLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java similarity index 91% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java index 1084932faf..82ba919970 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/KeyValueLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.chainlink.ChainLinkIO; -import nl.qsight.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; import org.json.simple.JSONObject; import java.util.Map; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java similarity index 90% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java index efc6625686..0d8155b453 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/RegexLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.chainlink.ChainLinkIO; -import nl.qsight.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; import org.json.simple.JSONObject; import java.util.Map; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java similarity index 92% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java index 63b6442182..29f2d0cdbf 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/SplitLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.chainlink.ChainLinkIO; -import nl.qsight.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; import org.json.simple.JSONObject; import java.util.Map; import java.util.regex.Pattern; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java similarity index 97% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java index 97788ebcc6..88540eccfb 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/links/io/TimestampLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.common.Constants; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -30,7 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static nl.qsight.common.Constants.TIMESTAMP; +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; public class TimestampLink extends ChainLinkIO { @@ -55,7 +57,7 @@ public void initPredefinedPatterns() { @SuppressWarnings("unchecked") private void addTimestampToMessage(JSONObject message, Long timestamp) { - message.put(TIMESTAMP, timestamp); + message.put(Constants.TIMESTAMP, timestamp); } private String extractLocalTime(String regex_result, String timestamp_mapping, String mapping_format) { diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java similarity index 91% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java index 3c1b4527ad..fd5effb8fc 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/ConfigUtils.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java @@ -15,12 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; - -import nl.qsight.chainlink.ChainLink; -import nl.qsight.common.Constants; -import nl.qsight.links.fields.RenderLink; -import nl.qsight.links.fields.SelectLink; +package org.apache.metron.parsers.contrib.utils; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.links.fields.SelectLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; import java.util.ArrayList; import java.util.HashMap; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java similarity index 97% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java index 71e5865c43..be0b989276 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/JSONUtils.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; +package org.apache.metron.parsers.contrib.utils; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.type.MapType; diff --git a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java similarity index 98% rename from metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java rename to metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java index f1d5aa02ad..6db0da9f05 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/nl/qsight/utils/StringUtils.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; +package org.apache.metron.parsers.contrib.utils; import java.util.HashMap; import java.util.Map; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java similarity index 93% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java index a6ef08882b..6140543a93 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainlink; +package org.apache.metron.parsers.contrib.chainlink; -import nl.qsight.chainlink.ChainLink; -import nl.qsight.links.fields.IdentityLink; +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java similarity index 95% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java index 6f9c11763a..6c5102ae38 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainlink/TestChainLinkIO.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainlink; +package org.apache.metron.parsers.contrib.chainlink; -import nl.qsight.chainlink.ChainLinkIO; -import nl.qsight.common.Constants; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.common.Constants; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java similarity index 85% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java index c17ac658fb..d62cf88f3d 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/chainparser/TestChainParser.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.chainparser; +package org.apache.metron.parsers.contrib.chainparser; -import nl.qsight.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -28,8 +30,8 @@ import java.util.List; import java.util.Map; -import static nl.qsight.common.Constants.ORIGINAL_STRING; -import static nl.qsight.common.Constants.TIMESTAMP; +import static org.apache.metron.parsers.contrib.common.Constants.ORIGINAL_STRING; +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; import static org.junit.Assert.*; public class TestChainParser { @@ -71,9 +73,9 @@ public void testRequiredFields() { // Check whether the state contains the required "original_string" message assertTrue("The parsed JSONObject should contain \"original_string\" field.", - state.containsKey(ORIGINAL_STRING)); + state.containsKey(Constants.ORIGINAL_STRING)); assertTrue("The parsed JSONObject should contain \"timestamp\" field.", - state.containsKey(TIMESTAMP)); + state.containsKey(Constants.TIMESTAMP)); } @Test @@ -88,7 +90,7 @@ public void testGetSetInitialLink() { public void testNoOriginalString() { // An exception should be thrown when a link removes the "original_string" field FieldRemoveLink link = new FieldRemoveLink(); - link.fieldToRemove = ORIGINAL_STRING; + link.fieldToRemove = Constants.ORIGINAL_STRING; this.parser.setInitialLink(link); this.parser.parse("".getBytes()); } @@ -97,7 +99,7 @@ public void testNoOriginalString() { public void testNoTimestamp() { // An exception should be thrown when a link removes the "timestamp" field FieldRemoveLink link = new FieldRemoveLink(); - link.fieldToRemove = TIMESTAMP; + link.fieldToRemove = Constants.TIMESTAMP; this.parser.setInitialLink(link); this.parser.parse("".getBytes()); } @@ -118,8 +120,8 @@ public void testEncodingGetterAndSetter() { public void testConfiguration() { Map link_1_config = new HashMap<>(); Map link_2_config = new HashMap<>(); - link_1_config.put("class", "nl.qsight.links.fields.IdentityLink"); - link_2_config.put("class", "nl.qsight.links.io.SplitLink"); + link_1_config.put("class", "org.apache.metron.parsers.contrib.links.fields.IdentityLink"); + link_2_config.put("class", "org.apache.metron.parsers.contrib.links.io.SplitLink"); Map selector = new HashMap<>(); selector.put("0", "first"); link_2_config.put("delimiter", "|"); diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java index e84e81d421..f74c909231 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestBlacklistLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.links.fields.BlacklistLink; +import org.apache.metron.parsers.contrib.links.fields.BlacklistLink; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java similarity index 97% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java index 931850fa6f..563e9292ce 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestNormalizeFieldLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java index 41f3c310de..b18d863a1b 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenameLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.links.fields.RenameLink; +import org.apache.metron.parsers.contrib.links.fields.RenameLink; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java similarity index 96% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java index 1c6e612110..77500b0de6 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestRenderLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java similarity index 90% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java index 1c8256cc5c..fe18ab09b9 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestSelectLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.common.Constants; -import nl.qsight.links.fields.SelectLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.SelectLink; +import org.apache.metron.parsers.contrib.common.Constants; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java similarity index 96% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java index 2b0796a5b9..1ecccf1509 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestTrimValueLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java index 39a5426d97..d828b8653b 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/fields/TestWhitelistLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.fields; +package org.apache.metron.parsers.contrib.links.fields; -import nl.qsight.links.fields.WhitelistLink; +import org.apache.metron.parsers.contrib.links.fields.WhitelistLink; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java similarity index 91% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java index 9b43a6fac2..352803c2dc 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestJSONDecodeLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; -import nl.qsight.common.Constants; -import nl.qsight.links.fields.WhitelistLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.WhitelistLink; import org.json.simple.JSONObject; import org.junit.After; import org.junit.Before; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java similarity index 98% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java index fd062472bf..6877957aac 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestKeyValueLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java similarity index 98% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java index 64fbdef3bf..f6b3f87c1d 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestRegexLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java similarity index 99% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java index 15a6bc196a..51a4e13d07 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestSplitLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java similarity index 99% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java index 89ab129115..d6e99e8c6d 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/links/io/TestTimestampLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.links.io; +package org.apache.metron.parsers.contrib.links.io; import org.json.simple.JSONObject; import org.junit.After; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java index eda0b1f908..c94e24cb1f 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestDhcp.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.parserconfig; +package org.apache.metron.parsers.contrib.parserconfig; public class TestDhcp extends TestParser { diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java similarity index 95% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java index a1b75ca1ab..d4c02424db 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestParser.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java @@ -15,11 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.parserconfig; - -import nl.qsight.chainlink.ChainLink; -import nl.qsight.chainparser.ChainParser; -import nl.qsight.utils.JSONUtils; +package org.apache.metron.parsers.contrib.parserconfig; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.utils.JSONUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.utils.JSONUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java index 1646fa5ec3..48d0075880 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestSuricata.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.parserconfig; +package org.apache.metron.parsers.contrib.parserconfig; public class TestSuricata extends TestParser { diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java similarity index 94% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java index 106f58b6ce..27fd3277b2 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/parserconfig/TestYaf.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.parserconfig; +package org.apache.metron.parsers.contrib.parserconfig; public class TestYaf extends TestParser { diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java similarity index 86% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java index 0a463b03cf..90f8caf221 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestConfigUtils.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java @@ -15,12 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; +package org.apache.metron.parsers.contrib.utils; -import nl.qsight.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import org.junit.Assert; import org.junit.Test; import java.io.IOException; @@ -28,7 +31,7 @@ import java.util.Map; import static junit.framework.TestCase.assertTrue; -import static nl.qsight.common.Constants.INPUT_MARKER; +import static org.apache.metron.parsers.contrib.common.Constants.INPUT_MARKER; import static org.junit.Assert.assertEquals; public class TestConfigUtils { @@ -64,7 +67,7 @@ public void testUnfoldInput() throws ParseException, IOException { assertEquals(RenderLink.class.getName(), configAutolink.get("class")); assertEquals("{{var1}}", configAutolink.get("template")); - assertEquals(INPUT_MARKER, configAutolink.get("output")); + Assert.assertEquals(Constants.INPUT_MARKER, configAutolink.get("output")); } } diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java similarity index 98% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java index 18c1ca188b..c5f76be70a 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestJSONUtils.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; +package org.apache.metron.parsers.contrib.utils; import org.json.simple.JSONObject; import org.json.simple.parser.ParseException; diff --git a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java similarity index 96% rename from metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java rename to metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java index 4e1c842404..eeba378022 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/nl/qsight/utils/TestStringUtils.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.qsight.utils; +package org.apache.metron.parsers.contrib.utils; import org.junit.Test; diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json index 57395684e7..8cba298ec0 100644 --- a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json @@ -11,16 +11,16 @@ ], "parsers": { "key_value_parser": { - "class": "nl.qsight.links.io.KeyValueLink", + "class": "org.apache.metron.parsers.contrib.links.io.KeyValueLink", "pair_delimiter": "|", "key_value_delimiter": ":", "valid_key_characters": "a-zA-z" }, "trim_whitespaces": { - "class" : "nl.qsight.links.fields.TrimValueLink" + "class" : "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" }, "parse_datetime": { - "class": "nl.qsight.links.io.TimestampLink", + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", "patterns": [ [ "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", @@ -36,7 +36,7 @@ "input": "{{TIME}}" }, "parse_macaddress": { - "class": "nl.qsight.links.io.RegexLink", + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", "pattern": "(Client-identifier: )([^\\|]++)", "selector": { "mac_address": "2" @@ -44,7 +44,7 @@ "input": "{{original_string}}" }, "parse_message_type": { - "class": "nl.qsight.links.io.RegexLink", + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", "pattern": "(DHCP message type: \\d \\|)([\\w]++)", "selector": { "message_type": "2" @@ -52,7 +52,7 @@ "input": "{{original_string}}" }, "parse_hostname": { - "class": "nl.qsight.links.io.RegexLink", + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", "pattern": "(Host name: )([^\\|]++)", "selector": { "hostname": "2" @@ -60,10 +60,10 @@ "input": "{{original_string}}" }, "normalize_fields": { - "class" : "nl.qsight.links.fields.NormalizeFieldLink" + "class" : "org.apache.metron.parsers.contrib.links.fields.NormalizeFieldLink" }, "rename_fields": { - "class": "nl.qsight.links.fields.RenameLink", + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", "rename": { "ciaddr": "ip_src_addr" } diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json index 633a3f5e28..f0897f4497 100644 --- a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json @@ -7,10 +7,10 @@ ], "parsers": { "parse_json": { - "class": "nl.qsight.links.io.JSONDecoderLink" + "class": "org.apache.metron.parsers.contrib.links.io.JSONDecoderLink" }, "parse_username": { - "class": "nl.qsight.links.io.RegexLink", + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", "pattern": "(?i)(user|username|log)[=:](\\w+)", "selector": { "username": "2" @@ -18,7 +18,7 @@ "input": "{{payload_printable}}" }, "rename_fields": { - "class": "nl.qsight.links.fields.RenameLink", + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", "rename": { "proto": "protocol", "dest_ip": "ip_dst_addr", @@ -28,7 +28,7 @@ } }, "parse_datetime": { - "class": "nl.qsight.links.io.TimestampLink", + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", "patterns": [ [ "([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json index 9799e4e727..831387a988 100644 --- a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json @@ -7,7 +7,7 @@ ], "parsers": { "values_splitter": { - "class": "nl.qsight.links.io.SplitLink", + "class": "org.apache.metron.parsers.contrib.links.io.SplitLink", "delimiter": "|", "selector": { "1": "start-time", @@ -38,7 +38,7 @@ } }, "parse_datetime": { - "class": "nl.qsight.links.io.TimestampLink", + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", "patterns": [ [ "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", @@ -54,10 +54,10 @@ "input": "{{end-time}}" }, "trim_whitespaces": { - "class" : "nl.qsight.links.fields.TrimValueLink" + "class" : "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" }, "rename_fields": { - "class": "nl.qsight.links.fields.RenameLink", + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", "rename": { "sip": "ip_src_addr", "sp": "ip_src_port", From 84776520fbd6787f15d6a23ba83725a2bb396d3a Mon Sep 17 00:00:00 2001 From: Renout Date: Tue, 27 Mar 2018 13:53:10 +0200 Subject: [PATCH 3/5] RegexFileLink + test created --- .../contrib/links/io/RegexFileLink.java | 165 ++++++++++++++++++ .../contrib/links/io/TestRegexFileLink.java | 64 +++++++ 2 files changed, 229 insertions(+) create mode 100644 metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java new file mode 100644 index 0000000000..6fffbbc5e8 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.hadoop.fs.Path; +import org.json.simple.JSONObject; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A link for reading a file contatining Regex patterns and parsing the variables when a match is found. + */ +public class RegexFileLink extends ChainLinkIO { + + private List patterns; + + public void configure(Map config) { + if (config.containsKey("file")) { + assert config.get("file") instanceof String; + this.readFile((String) config.get("file")); + } + if (config.containsKey("patterns")) { + assert config.get("patterns") instanceof List; + this.setPatterns((List) config.get("patterns")); + } + } + + + @Override + public Object parseInputField(String input) { + JSONObject result = new JSONObject(); + + for (String pattern : this.patterns) { + JSONObject variables = extractVariables(input, pattern); + System.out.println(pattern); + if (variables.keySet().size() > 0) { + System.out.println("Match found."); + return variables; + } + } + + return result; + } + + public void setPatterns(List patterns) { + this.patterns = patterns; + } + + public JSONObject extractVariables(String logline, String pattern) { + JSONObject vars = new JSONObject(); + + // Skip some lines + if (pattern.startsWith("[")) return new JSONObject(); + if (pattern.trim().length() == 0) return new JSONObject(); + + if (pattern.substring(0, 1).equals("\"") && pattern.substring(pattern.length() - 1).equals("\"")) { + pattern = pattern.substring(1, pattern.length() - 1); + } + + // Make sure the patterns do not start with ?P + pattern = pattern.replaceAll("\\?P\\<", "\\?\\<"); + + // Find the non-preprocessed group names + List originalGroups = findGroups(pattern, "\\?\\<([^>]+)\\>"); + + // Clean the groups such that these can be used by the Regex parser + pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); + pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); + pattern = pattern.replaceAll("\\?\\(([a-zA-Z0-9_]+)\\)", "\\\\k<$1>"); + pattern = pattern.replaceAll("\\\\k\\<([^>]*)[^a-zA-Z0-9>]([^>]*)\\>", "\\\\k<$1$2>"); + + // Custom rules + pattern = pattern.replaceAll("[\\\\]{1}Type", "Type"); + pattern = pattern.replaceAll("[\\\\]{1}IPV4", "IPV4"); + pattern = pattern.replaceAll("[\\\\]{1}User", "User"); + + System.out.println(pattern); + + // Find all group names + List groups = findGroups(pattern, "\\?\\<([a-zA-Z0-9]+)\\>"); + + // Find all groups + Pattern regexPattern = Pattern.compile(pattern); + Matcher matcher = regexPattern.matcher(logline); + if (matcher.matches()) { + for (String group : groups) { + String value = matcher.group(group); + + // Find the original group name + int index = groups.indexOf(group); + String varName = originalGroups.get(index); + + vars.put(varName, value); + } + } + + // If the number of groups are not correct, return an empty JSON Object + if (vars.keySet().size() != groups.size()) return new JSONObject(); + + return vars; + } + + private List findGroups(String input, String pattern) { + // Find all group names + Pattern groupPattern = Pattern.compile(pattern); + Matcher groupMatcher = groupPattern.matcher(input); + List groups = new ArrayList<>(); + while (groupMatcher.find()) { + groups.add(groupMatcher.group(1)); + } + + return groups; + } + + public void readFile(String filepath) { + List patterns = new ArrayList<>(); + + Path path = new Path(filepath); + try { + FileSystem fs = FileSystem.get(URI.create(filepath), new Configuration()); + if (fs.exists(path)) { + InputStream inputStream = fs.open(path); + java.util.Scanner scanner = new java.util.Scanner(inputStream).useDelimiter("\n"); + while (scanner.hasNext()) { + String pattern = scanner.next(); + if (pattern.length() > 0) { + patterns.add(pattern); + } + } + } else { + throw new IllegalStateException("[Metron] Could not find the file: " + filepath); + } + } catch (IOException e) { + throw new IllegalStateException("[Metron] Could not load the file: " + filepath); + } + + this.setPatterns(patterns); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java new file mode 100644 index 0000000000..a3a357bb56 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRegexFileLink { + + private RegexFileLink link; + + @Before + public void setUp() { + this.link = new RegexFileLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testRegexFileLink() { + List patterns = new ArrayList<>(); + patterns.add("NUM:(?P\\d+)"); + String logline = "NUM:1234"; + this.link.setPatterns(patterns); + JSONObject output = (JSONObject) this.link.parseInputField(logline); + assertTrue(output.containsKey("number")); + assertEquals("1234", output.get("number")); + + patterns.add("^AV\\s-\\sAlert\\s-\\s(?P\\d+)"); + logline = "AV - Alert - 1234"; + this.link.setPatterns(patterns); + output = (JSONObject) this.link.parseInputField(logline); + System.out.println(output); + } + +} From 45d5a9b3966cfea4db1c8eb3a4a5ce4e7f6bc157 Mon Sep 17 00:00:00 2001 From: Kevin Jacobs Date: Tue, 27 Mar 2018 16:31:16 +0200 Subject: [PATCH 4/5] Added OSSEC and the RegexFileLink. --- .../metron-parsers-contrib/README.md | 18 ++++ .../contrib/links/io/RegexFileLink.java | 36 ++------ .../contrib/links/io/TestRegexFileLink.java | 91 ++++++++++++++++++- .../contrib/parserconfig/TestOssec.java | 26 ++++++ .../src/test/resources/ossec/config.json | 43 +++++++++ .../src/test/resources/ossec/data_input | 1 + .../src/test/resources/ossec/data_output | 1 + 7 files changed, 187 insertions(+), 29 deletions(-) create mode 100644 metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input create mode 100644 metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output diff --git a/metron-platform/metron-parsers-contrib/README.md b/metron-platform/metron-parsers-contrib/README.md index d0433ea50b..5a23240e20 100644 --- a/metron-platform/metron-parsers-contrib/README.md +++ b/metron-platform/metron-parsers-contrib/README.md @@ -207,6 +207,24 @@ The pair delimiter would be `|` and the key-value delimiter is `=`. The followin The "pair_delimiter" argument specifies which pair delimiter is used. The "key_value_delimiter" argument specifies the key-value delimiter and the "valid_key_characters" argument specifies the characters of which a key exists. The last argument is a substring of the regular expression for detecting keys and is required for performance issues. The value "A-Z" refers to the fact that the keys consist only of uppercase characters. +#### Regex file link + +The Regex file link reads Regex patterns from a file (line separated) and tries to find a match. When there is a match, all named variables (like `(?\d+)`) are stored in the output JSON object. + +Instead of specifying a file to load the patterns from, it is also possible to specify the `patterns` field which should be an array of patterns. It is possible to load a file from HDFS (by specifying the `hdfs://` prefix). The file is specified in the `file` field. + +```json +{ + "chain": ["parse_regexfile"], + "parsers": { + "keyvalue": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexFileLink", + "file": "hdfs://my_regex_file.txt" + } + } +} +``` + #### Regex link The regex link executes a regular expression on the input field and uses the first found result for the creation of the output object. diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java index 6fffbbc5e8..48ab331f83 100644 --- a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java @@ -57,9 +57,7 @@ public Object parseInputField(String input) { for (String pattern : this.patterns) { JSONObject variables = extractVariables(input, pattern); - System.out.println(pattern); if (variables.keySet().size() > 0) { - System.out.println("Match found."); return variables; } } @@ -71,6 +69,9 @@ public void setPatterns(List patterns) { this.patterns = patterns; } + public List getPatterns() { return this.patterns; } + + @SuppressWarnings("unchecked") public JSONObject extractVariables(String logline, String pattern) { JSONObject vars = new JSONObject(); @@ -78,36 +79,17 @@ public JSONObject extractVariables(String logline, String pattern) { if (pattern.startsWith("[")) return new JSONObject(); if (pattern.trim().length() == 0) return new JSONObject(); - if (pattern.substring(0, 1).equals("\"") && pattern.substring(pattern.length() - 1).equals("\"")) { - pattern = pattern.substring(1, pattern.length() - 1); - } - - // Make sure the patterns do not start with ?P - pattern = pattern.replaceAll("\\?P\\<", "\\?\\<"); - // Find the non-preprocessed group names List originalGroups = findGroups(pattern, "\\?\\<([^>]+)\\>"); - // Clean the groups such that these can be used by the Regex parser - pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); - pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); - pattern = pattern.replaceAll("\\?\\(([a-zA-Z0-9_]+)\\)", "\\\\k<$1>"); - pattern = pattern.replaceAll("\\\\k\\<([^>]*)[^a-zA-Z0-9>]([^>]*)\\>", "\\\\k<$1$2>"); - - // Custom rules - pattern = pattern.replaceAll("[\\\\]{1}Type", "Type"); - pattern = pattern.replaceAll("[\\\\]{1}IPV4", "IPV4"); - pattern = pattern.replaceAll("[\\\\]{1}User", "User"); - - System.out.println(pattern); - // Find all group names List groups = findGroups(pattern, "\\?\\<([a-zA-Z0-9]+)\\>"); // Find all groups - Pattern regexPattern = Pattern.compile(pattern); - Matcher matcher = regexPattern.matcher(logline); - if (matcher.matches()) { + Pattern regexPattern = Pattern.compile(pattern.trim(), Pattern.CASE_INSENSITIVE); + Matcher matcher = regexPattern.matcher(logline.trim()); + + if (matcher.find()) { for (String group : groups) { String value = matcher.group(group); @@ -153,10 +135,10 @@ public void readFile(String filepath) { } } } else { - throw new IllegalStateException("[Metron] Could not find the file: " + filepath); + throw new IllegalStateException("Could not find the file: " + filepath); } } catch (IOException e) { - throw new IllegalStateException("[Metron] Could not load the file: " + filepath); + throw new IllegalStateException("Could not load the file: " + filepath); } this.setPatterns(patterns); diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java index a3a357bb56..d9556c9964 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java @@ -22,10 +22,16 @@ import org.junit.Before; import org.junit.Test; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -54,11 +60,92 @@ public void testRegexFileLink() { assertTrue(output.containsKey("number")); assertEquals("1234", output.get("number")); - patterns.add("^AV\\s-\\sAlert\\s-\\s(?P\\d+)"); - logline = "AV - Alert - 1234"; + patterns.add("AV\\s-\\sAlert\\s-\\s\"(?P\\d+)\""); + logline = "AV - Alert - \"1234\""; this.link.setPatterns(patterns); output = (JSONObject) this.link.parseInputField(logline); System.out.println(output); } + @Test + public void testCompiledRegex() { + String file = "file:///C:/Users/kevjac/Desktop/ossec_regex_compiled.txt"; + this.link.readFile(file); + + String data = "AV - Alert - \"1498521603\" --> RID: \"18149\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"C1238C89R$\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1238C89R$: EXID: 123.nl: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";"; + String[] lines = data.split("[\\r\\n]+"); + for (String line: lines) { + line = line.trim(); + JSONObject output = (JSONObject) this.link.parseInputField(line); + System.out.println(line); + System.out.println(output.keySet().size()); + System.out.println(output); + } + //this.link.parseInputField("AV - Alert - \"1498521603\" --> RID: \"18107\"; RL: \"3\"; RG: \"windows,authentication_success,\"; RC: \"Windows Logon Success.\"; USER: \"SVC00014\"; SRCIP: \"None\"; HOSTNAME: \"(xidaddsp10-exid-umcn-nl) 131.174.163.148->WinEvtLog\"; LOCATION: \"(xidaddsp10-exid-umcn-nl) 131.174.163.148->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:58 WinEvtLog: Security: AUDIT_SUCCESS(4624): Microsoft-Windows-Security-Auditing: SVC00014: EXID: XIDADDSP10.exid.umcn.nl: An account was successfully logged on. Subject: Security ID: S-1-0-0 Account Name: - Account Domain: - Logon ID: 0x0 Logon Type: 3 New Logon: Security ID: S-1-5-21-2303058550-3105691012-4221217832-1694 Account Name: SVC00014 Account Domain: EXID Logon ID: 0x222d4cfa Logon GUID: {75763528-BF37-2C36-640E-B8B179973B8B} Process Information: Process ID: 0x0 Process Name: - Network Information: Workstation Name: Source Network Address: 131.174.244.80 Source Port: 49047 Detailed Authentication Information: Logon Process: Kerberos Authentication Package: Kerberos Transited Services: - Package Name (NTLM only): - Key Length: 0 This event is generated when a logon session is created. It is generated on the computer that was accessed. [END]\";"); + } + + @Test + public void testTesting() { + final String regex = "^AV\\s+\\-\\sAlert\\s+\\-\\s\\\"(?\\d+)\\\"\\s\\-\\->\\sRID\\:\\s\\\"(?\\d+)\\\"\\;\\s+RL\\:\\s+\\\"(?\\d+)\\\"\\;\\s+RG\\:\\s+\\\"(?[^\\\"]*)\\\"\\;\\s+RC\\:\\s+\\\"(?[^\\\"]*)\\\";.*?HOSTNAME\\:\\s*\"?\\((?[^\\)]*)\\)\\s(?\\S+)->.*?AUDIT_SUCCESS\\((?(4634|4647))\\).*?Account\\s+Name\\:\\s+(?.*?)\\s*Account\\s+Domain\\:\\s+(?.*?)\\s*Logon\\s+ID\\:\\s+(?\\S+)(\\s+Logon\\s+Type\\:\\s+(?\\d+))?.*"; + final String string = "AV - Alert - \"1498521603\" --> RID: \"18149\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"C1238C89R$\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1238C89R$: EXID: 123.nl: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";\n"; + + final Pattern pattern = Pattern.compile(regex); + final Matcher matcher = pattern.matcher(string); + + while (matcher.find()) { + System.out.println("Full match: " + matcher.group(0)); + for (int i = 1; i <= matcher.groupCount(); i++) { + System.out.println("Group " + i + ": " + matcher.group(i)); + } + } + } + + @Test + public void testChangeFile() throws IOException { + String file = "C:/Users/kevjac/Desktop/ossec_regex.txt"; + + // Open the file + FileInputStream fstream = new FileInputStream(file); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + + String strLine; + String output = ""; + + //Read File Line By Line + while ((strLine = br.readLine()) != null) { + // Print the content on the console + String line = strLine; + if (line.trim().length() == 0 || line.trim().startsWith("[")) { + //output += line + "\n"; + } else { + + String pattern = line; + if (pattern.substring(0, 1).equals("\"") && pattern.substring(pattern.length() - 1).equals("\"")) { + pattern = pattern.substring(1, pattern.length() - 1); + } + + // Make sure the patterns do not start with ?P + pattern = pattern.replaceAll("\\?P\\<", "\\?\\<"); + + // Clean the groups such that these can be used by the Regex parser + pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); + pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); + pattern = pattern.replaceAll("\\?\\(([a-zA-Z0-9_]+)\\)", "\\\\k<$1>"); + pattern = pattern.replaceAll("\\\\k\\<([^>]*)[^a-zA-Z0-9>]([^>]*)\\>", "\\\\k<$1$2>"); + + // Custom rules + pattern = pattern.replaceAll("[\\\\]{1}Type", "Type"); + pattern = pattern.replaceAll("[\\\\]{1}IPV4", "IPV4"); + pattern = pattern.replaceAll("[\\\\]{1}User", "User"); + + output += pattern + "\n"; + } + } + + //Close the input stream + br.close(); + + System.out.println(output); + } + } diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java new file mode 100644 index 0000000000..a7379dcab8 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestOssec extends TestParser { + + @Override + public String getFolder() { + return "ossec"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json new file mode 100644 index 0000000000..a1d44d9305 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json @@ -0,0 +1,43 @@ +{ + "chain": [ + "regex_file_parser", + "parse_timestamp", + "rename_fields" + ], + "parsers": { + "regex_file_parser": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexFileLink", + "patterns": [ + "^AV\\s+\\-\\sAlert\\s+\\-\\s\\\"(?\\d+)\\\"\\s\\-\\->\\sRID\\:\\s\\\"(?\\d+)\\\"\\;\\s+RL\\:\\s+\\\"(?\\d+)\\\"\\;\\s+RG\\:\\s+\\\"(?[^\\\"]*)\\\"\\;\\s+RC\\:\\s+\\\"(?[^\\\"]*)\\\";.*?HOSTNAME\\:\\s*\"?\\((?[^\\)]*)\\)\\s(?\\S+)->.*?AUDIT_SUCCESS\\((?(4634|4647))\\).*?Account\\s+Name\\:\\s+(?.*?)\\s*Account\\s+Domain\\:\\s+(?.*?)\\s*Logon\\s+ID\\:\\s+(?\\S+)(\\s+Logon\\s+Type\\:\\s+(?\\d+))?.*" + ], + "input": "{{original_string}}" + }, + "parse_timestamp": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + [ + "timestamp", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})", + "yyyy MM dd HH mm ss", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss Z", + "newest" + ] + ], + "input": "{{original_string}}" + }, + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "rename": { + "srcip": "ip_src_addr", + "srcport": "ip_src_port" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input new file mode 100644 index 0000000000..f425f8c60e --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input @@ -0,0 +1 @@ +AV - Alert - "1498521603" --> RID: "1234"; RL: "3"; RG: "windows,"; RC: "Windows User Logoff."; USER: "1234"; SRCIP: "None"; HOSTNAME: "(123-nl) 123.123.123.123->WinEvtLog"; LOCATION: "(123-nl) 123.123.123.123->WinEvtLog"; EVENT: "[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1234: EXID: 123.com: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer." 4646,1[END]"; \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output new file mode 100644 index 0000000000..12af202ba7 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output @@ -0,0 +1 @@ +{"rulecomment":"Windows User Logoff.","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","timezone_available":1,"logontype":"3","hostname":"123-nl","datetime":"2017-06-27 00:00:03.000 +0000","original_string":"AV - Alert - \"1498521603\" --> RID: \"1234\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"1234\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1234: EXID: 123.com: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";","wineventid":"4634","original_timestamp":"2017-06-27 00:00:03.000","logonid":"0x222d4cc7","rulelevel":"3","domain":"EXID","ruleid":"1234","rulegroup":"windows,","winip":"123.123.123.123","timestamp":1498521603000,"username":"myusername"} \ No newline at end of file From e182f070ca707888be9c2860a6017f07a572dfbe Mon Sep 17 00:00:00 2001 From: Kevin Jacobs Date: Tue, 27 Mar 2018 16:34:48 +0200 Subject: [PATCH 5/5] Simplified RegexFileLink unit tests. --- .../contrib/links/io/TestRegexFileLink.java | 89 +------------------ 1 file changed, 1 insertion(+), 88 deletions(-) diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java index d9556c9964..61f212ce6f 100644 --- a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java @@ -53,99 +53,12 @@ public void tearDown() { @Test public void testRegexFileLink() { List patterns = new ArrayList<>(); - patterns.add("NUM:(?P\\d+)"); + patterns.add("NUM:(?\\d+)"); String logline = "NUM:1234"; this.link.setPatterns(patterns); JSONObject output = (JSONObject) this.link.parseInputField(logline); assertTrue(output.containsKey("number")); assertEquals("1234", output.get("number")); - - patterns.add("AV\\s-\\sAlert\\s-\\s\"(?P\\d+)\""); - logline = "AV - Alert - \"1234\""; - this.link.setPatterns(patterns); - output = (JSONObject) this.link.parseInputField(logline); - System.out.println(output); - } - - @Test - public void testCompiledRegex() { - String file = "file:///C:/Users/kevjac/Desktop/ossec_regex_compiled.txt"; - this.link.readFile(file); - - String data = "AV - Alert - \"1498521603\" --> RID: \"18149\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"C1238C89R$\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1238C89R$: EXID: 123.nl: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";"; - String[] lines = data.split("[\\r\\n]+"); - for (String line: lines) { - line = line.trim(); - JSONObject output = (JSONObject) this.link.parseInputField(line); - System.out.println(line); - System.out.println(output.keySet().size()); - System.out.println(output); - } - //this.link.parseInputField("AV - Alert - \"1498521603\" --> RID: \"18107\"; RL: \"3\"; RG: \"windows,authentication_success,\"; RC: \"Windows Logon Success.\"; USER: \"SVC00014\"; SRCIP: \"None\"; HOSTNAME: \"(xidaddsp10-exid-umcn-nl) 131.174.163.148->WinEvtLog\"; LOCATION: \"(xidaddsp10-exid-umcn-nl) 131.174.163.148->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:58 WinEvtLog: Security: AUDIT_SUCCESS(4624): Microsoft-Windows-Security-Auditing: SVC00014: EXID: XIDADDSP10.exid.umcn.nl: An account was successfully logged on. Subject: Security ID: S-1-0-0 Account Name: - Account Domain: - Logon ID: 0x0 Logon Type: 3 New Logon: Security ID: S-1-5-21-2303058550-3105691012-4221217832-1694 Account Name: SVC00014 Account Domain: EXID Logon ID: 0x222d4cfa Logon GUID: {75763528-BF37-2C36-640E-B8B179973B8B} Process Information: Process ID: 0x0 Process Name: - Network Information: Workstation Name: Source Network Address: 131.174.244.80 Source Port: 49047 Detailed Authentication Information: Logon Process: Kerberos Authentication Package: Kerberos Transited Services: - Package Name (NTLM only): - Key Length: 0 This event is generated when a logon session is created. It is generated on the computer that was accessed. [END]\";"); - } - - @Test - public void testTesting() { - final String regex = "^AV\\s+\\-\\sAlert\\s+\\-\\s\\\"(?\\d+)\\\"\\s\\-\\->\\sRID\\:\\s\\\"(?\\d+)\\\"\\;\\s+RL\\:\\s+\\\"(?\\d+)\\\"\\;\\s+RG\\:\\s+\\\"(?[^\\\"]*)\\\"\\;\\s+RC\\:\\s+\\\"(?[^\\\"]*)\\\";.*?HOSTNAME\\:\\s*\"?\\((?[^\\)]*)\\)\\s(?\\S+)->.*?AUDIT_SUCCESS\\((?(4634|4647))\\).*?Account\\s+Name\\:\\s+(?.*?)\\s*Account\\s+Domain\\:\\s+(?.*?)\\s*Logon\\s+ID\\:\\s+(?\\S+)(\\s+Logon\\s+Type\\:\\s+(?\\d+))?.*"; - final String string = "AV - Alert - \"1498521603\" --> RID: \"18149\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"C1238C89R$\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1238C89R$: EXID: 123.nl: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";\n"; - - final Pattern pattern = Pattern.compile(regex); - final Matcher matcher = pattern.matcher(string); - - while (matcher.find()) { - System.out.println("Full match: " + matcher.group(0)); - for (int i = 1; i <= matcher.groupCount(); i++) { - System.out.println("Group " + i + ": " + matcher.group(i)); - } - } - } - - @Test - public void testChangeFile() throws IOException { - String file = "C:/Users/kevjac/Desktop/ossec_regex.txt"; - - // Open the file - FileInputStream fstream = new FileInputStream(file); - BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); - - String strLine; - String output = ""; - - //Read File Line By Line - while ((strLine = br.readLine()) != null) { - // Print the content on the console - String line = strLine; - if (line.trim().length() == 0 || line.trim().startsWith("[")) { - //output += line + "\n"; - } else { - - String pattern = line; - if (pattern.substring(0, 1).equals("\"") && pattern.substring(pattern.length() - 1).equals("\"")) { - pattern = pattern.substring(1, pattern.length() - 1); - } - - // Make sure the patterns do not start with ?P - pattern = pattern.replaceAll("\\?P\\<", "\\?\\<"); - - // Clean the groups such that these can be used by the Regex parser - pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); - pattern = pattern.replaceAll("\\<([^>]*)[^a-zA-Z0-9>]{1,}([^>]*)\\>", "<$1$2>"); - pattern = pattern.replaceAll("\\?\\(([a-zA-Z0-9_]+)\\)", "\\\\k<$1>"); - pattern = pattern.replaceAll("\\\\k\\<([^>]*)[^a-zA-Z0-9>]([^>]*)\\>", "\\\\k<$1$2>"); - - // Custom rules - pattern = pattern.replaceAll("[\\\\]{1}Type", "Type"); - pattern = pattern.replaceAll("[\\\\]{1}IPV4", "IPV4"); - pattern = pattern.replaceAll("[\\\\]{1}User", "User"); - - output += pattern + "\n"; - } - } - - //Close the input stream - br.close(); - - System.out.println(output); } }