From 8ee9eb8007ce3c0bbca553e8c4b5d861300d1a00 Mon Sep 17 00:00:00 2001 From: Simon Elliston Ball Date: Wed, 30 Aug 2017 14:35:39 -0400 Subject: [PATCH 1/5] METRON-1139 Fixed service advisor profilerHost variable closes apache/incubator-metron#722 --- .../resources/common-services/METRON/CURRENT/service_advisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py index 2fb1ab09f2..1a9d195ac5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py @@ -43,7 +43,7 @@ def getServiceComponentLayoutValidations(self, services, hosts): metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0] metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0] - metronProfilerMaster = self.getHosts(componentsList, "METRON_PROFILER")[0] + metronProfilerHost = self.getHosts(componentsList, "METRON_PROFILER")[0] metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0] metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0] From cd14cbeefec6afc7aee9efcc288c9482d67d9657 Mon Sep 17 00:00:00 2001 From: anandsubbu Date: Thu, 31 Aug 2017 08:14:12 -0400 Subject: [PATCH 2/5] METRON-1137 Build RPM for Metron MaaS as a part of rpm-docker packaging (anandsubbu via nickwallen) closes apache/metron#719 --- .../docker/rpm-docker/SPECS/metron.spec | 23 +++++++++++++++++++ .../packaging/docker/rpm-docker/pom.xml | 6 +++++ 2 files changed, 29 insertions(+) diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 94c7e05207..5a888778f7 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -53,6 +53,7 @@ Source8: metron-profiler-%{full_version}-archive.tar.gz Source9: metron-rest-%{full_version}-archive.tar.gz Source10: metron-config-%{full_version}-archive.tar.gz Source11: metron-management-%{full_version}-archive.tar.gz +Source12: metron-maas-service-%{full_version}-archive.tar.gz %description Apache Metron provides a scalable advanced security analytics framework @@ -87,6 +88,7 @@ tar -xzf %{SOURCE8} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE9} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE10} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE11} -C %{buildroot}%{metron_home} +tar -xzf %{SOURCE12} -C %{buildroot}%{metron_home} install %{buildroot}%{metron_home}/bin/metron-rest %{buildroot}/etc/init.d/ install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/ @@ -426,7 +428,28 @@ chkconfig --del metron-management-ui # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +%package maas-service +Summary: Metron MaaS service +Group: Application/Internet +Provides: maas-service = %{version} + +%description maas-service +This package install the Metron MaaS Service files %{metron_home} + +%files maas-service +%defattr(-,root,root,755) +%dir %{metron_root} +%dir %{metron_home} +%dir %{metron_home}/bin +%{metron_home}/bin/maas_service.sh +%{metron_home}/bin/maas_deploy.sh +%attr(0644,root,root) %{metron_home}/lib/metron-maas-service-%{full_version}-uber.jar + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + %changelog +* Tue Aug 29 2017 Apache Metron - 0.4.1 +- Add Metron MaaS service * Thu Jun 29 2017 Apache Metron - 0.4.1 - Add Metron Management jar * Thu May 15 2017 Apache Metron - 0.4.0 diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 749acd20dd..b6a9ce3e0d 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -173,6 +173,12 @@ *.tar.gz + + ${metron_dir}/metron-analytics/metron-maas-service/target/ + + *.tar.gz + + From 1a4f19c86ed94742c08e8e264a7bb45ff9ad27a6 Mon Sep 17 00:00:00 2001 From: nickwallen Date: Thu, 31 Aug 2017 10:10:53 -0400 Subject: [PATCH 3/5] METRON-1117 Filter Functions Returned by `%functions` (nickwallen) closes apache/metron#704 --- metron-stellar/stellar-common/README.md | 10 +++- .../stellar/common/shell/StellarShell.java | 55 ++++++++++++------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 8746e60e33..926132e70c 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -1052,7 +1052,7 @@ The REPL has a set of magic commands that provide the REPL user with information #### `%functions` -This command lists all functions resolvable in the Stellar environment. Stellar searches the classpath for Stellar functions. This can make it difficult in some cases to understand which functions are resolvable. +This command lists all functions resolvable in the Stellar environment. ``` [Stellar]>>> %functions @@ -1066,7 +1066,13 @@ STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR -[Stellar]>>> +``` + +The list of functions returned can also be filtered by passing an argument. Only the functions containing the argument as a substring will be returned. + +``` +[Stellar]>>> %functions NET +IN_SUBNET ``` #### `%vars` diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 0d2f0c363c..a860778b98 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -283,36 +284,44 @@ private void handleStellar(String expression) { } /** - * Handles user interaction when executing a Magic command. + * Executes a magic expression. * @param rawExpression The expression to execute. */ private void handleMagic( String rawExpression) { - String expression = rawExpression.trim(); - if(MAGIC_FUNCTIONS.equals(expression)) { + String[] expression = rawExpression.trim().split(" "); - // list all functions + String command = expression[0]; + if(MAGIC_FUNCTIONS.equals(command)) { + + // if '%functions FOO' then show only functions that contain 'FOO' + Predicate nameFilter = (name -> true); + if(expression.length > 1) { + nameFilter = (name -> name.contains(expression[1])); + } + + // list available functions String functions = StreamSupport .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) .map(info -> String.format("%s", info.getName())) + .filter(nameFilter) .sorted() .collect(Collectors.joining(", ")); writeLine(functions); - } else if(MAGIC_VARS.equals(expression)) { + } else if(MAGIC_VARS.equals(command)) { // list all variables - executor.getVariables() .forEach((k,v) -> writeLine(String.format("%s = %s", k, v))); } else { - writeLine(ERROR_PROMPT + "undefined magic command: " + expression); + writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression); } } /** - * Handles user interaction when executing a doc command. - * @param expression The expression to execute. + * Executes a doc expression. + * @param expression The doc expression to execute. */ private void handleDoc(String expression) { @@ -324,6 +333,17 @@ private void handleDoc(String expression) { .forEach(doc -> write(doc)); } + /** + * Executes a quit. + */ + private void handleQuit() { + try { + console.stop(); + } catch (Throwable e) { + e.printStackTrace(); + } + } + /** * Formats the Stellar function info object into a readable string. * @param info The stellar function info object. @@ -380,16 +400,12 @@ public int execute(ConsoleOperation output) throws InterruptedException { handleDoc(expression); } else if (expression.equals("quit")) { - try { - console.stop(); - } catch (Throwable e) { - e.printStackTrace(); - } - } - else if(expression.charAt(0) == '#') { - return 0; - } - else { + handleQuit(); + + } else if(expression.charAt(0) == '#') { + return 0; // comment, do nothing + + } else { handleStellar(expression); } } @@ -424,7 +440,6 @@ else if(isMagic(lastToken)) { } } } - } private static String stripOff(String baseString, String lastBit) { From c73c7af1dc3cb6502517bc7ac1d37c631ec3dbe5 Mon Sep 17 00:00:00 2001 From: nickwallen Date: Thu, 31 Aug 2017 10:38:21 -0400 Subject: [PATCH 4/5] METRON-1132 Enhance Profiler Debug Functions (nickwallen) closes apache/metron#716 --- .../metron-profiler-client/README.md | 21 +++-- .../client/stellar/ProfilerFunctions.java | 35 +++++-- .../client/stellar/ProfilerFunctionsTest.java | 93 ++++++++++++++++++- .../metron/profiler/StandAloneProfiler.java | 40 ++++++++ metron-analytics/metron-profiler/README.md | 2 + 5 files changed, 172 insertions(+), 19 deletions(-) diff --git a/metron-analytics/metron-profiler-client/README.md b/metron-analytics/metron-profiler-client/README.md index a8b5a554d6..27aa3f47b2 100644 --- a/metron-analytics/metron-profiler-client/README.md +++ b/metron-analytics/metron-profiler-client/README.md @@ -418,8 +418,11 @@ Follow these steps in the Stellar REPL to see how it can be used to help create ``` [Stellar]>>> profiler := PROFILER_INIT(conf) [Stellar]>>> profiler - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 + Profiler{1 profile(s), 0 messages(s), 0 route(s)} ``` + The profiler itself will show the number of profiles defined, the number of messages applied, and the number of routes taken. + + A route is defined when a message is applied to a specific profile. If a message is applied and not needed by any profile, then there are no routes. If a message is needed by one profile, then one route has been defined. If a message is needed by two profiles, then two routes have been defined. 1. Create a message to simulate the type of telemetry that you expect to be profiled. As an example, in the editor copy/paste the JSON below. ``` @@ -436,14 +439,18 @@ Follow these steps in the Stellar REPL to see how it can be used to help create 1. Apply some telemetry messages to your profiles. The following applies the same message 3 times. ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 - + Profiler{1 profile(s), 1 messages(s), 1 route(s)} + ``` + ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 - + Profiler{1 profile(s), 2 messages(s), 2 route(s)} + ``` + ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 + Profiler{1 profile(s), 3 messages(s), 3 route(s)} ``` + + It is also possible to apply multiple messages at once. This is useful when testing against a larger set of data. To do this, create a string that contains a JSON array of messages and pass that to the `PROFILER_APPLY` function. 1. Flush the Profiler to see what has been calculated. A flush is what occurs at the end of each 15 minute period in the Profiler. The result is a list of profile measurements. Each measurement is a map containing detailed information about the profile data that has been generated. ``` @@ -455,4 +462,4 @@ Follow these steps in the Stellar REPL to see how it can be used to help create This profile simply counts the number of messages by IP source address. Notice that the value is '3' for the entity '10.0.0.1' as we applied 3 messages with an 'ip_src_addr' of '10.0.0.1'. There will always be one measurement for each [profile, entity] pair. -1. If you are unhappy with the data that has been generated, then 'wash, rinse and repeat' this process. After you are satisfied with the data being generated by the profile, then follow the [Getting Started](../metron-profiler#getting-started) guide to use the profile against your live, streaming data in a Metron cluster. \ No newline at end of file +1. If you are unhappy with the data that has been generated, then 'wash, rinse and repeat' this process. Once you are happy with the profile that was created, follow the [Getting Started](../metron-profiler#getting-started) guide to use the profile against your live, streaming data in a Metron cluster. \ No newline at end of file diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java index 827e1c4ac8..8df5ca8638 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java @@ -28,6 +28,8 @@ import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.storm.shade.org.apache.commons.lang.ClassUtils; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.slf4j.LoggerFactory; @@ -107,7 +109,7 @@ public Object apply(List args, Context context) { name="APPLY", description="Apply a message to a local profile runner.", params={ - "message", "The message to apply.", + "message(s)", "The message to apply. A JSON list can be used to apply multiple messages.", "profiler", "A local profile runner returned by PROFILER_INIT." }, returns="The local profile runner." @@ -129,16 +131,33 @@ public boolean isInitialized() { @Override public Object apply(List args, Context context) throws ParseException { - // user must provide the json telemetry message + // user must provide the message as a string String arg0 = Util.getArg(0, String.class, args); if(arg0 == null) { throw new IllegalArgumentException(format("expected string, found null")); } - // parse the message - JSONObject message; + // there could be one or more messages + List messages = new ArrayList<>(); try { - message = (JSONObject) parser.parse(arg0); + Object parsedArg0 = parser.parse(arg0); + if(parsedArg0 instanceof JSONObject) { + // if there is only one message + messages.add((JSONObject) parsedArg0); + + } else if(parsedArg0 instanceof JSONArray) { + // there are multiple messages + JSONArray jsonArray = (JSONArray) parsedArg0; + for(Object json: jsonArray) { + if(json instanceof JSONObject) { + messages.add((JSONObject) json); + + } else { + throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject", + ClassUtils.getShortClassName(json, "null"))); + } + } + } } catch(org.json.simple.parser.ParseException e) { throw new IllegalArgumentException("invalid message", e); @@ -147,10 +166,12 @@ public Object apply(List args, Context context) throws ParseException { // user must provide the stand alone profiler StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args); try { - profiler.apply(message); + for(JSONObject message : messages) { + profiler.apply(message); + } } catch(ExecutionException e) { - throw new IllegalArgumentException(e); + throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e); } return profiler; diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java index 9cc8046b74..bad3efe60d 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java @@ -57,6 +57,28 @@ public class ProfilerFunctionsTest { @Multiline private String message; + /** + * [ + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * }, + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * }, + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * } + * ] + */ + @Multiline + private String messages; + /** * { * "profiles": [ @@ -108,7 +130,9 @@ public void testProfilerInitNoProfiles() { state.put("config", "{ \"profiles\" : [] }"); StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(0, profiler.getConfig().getProfiles().size()); + assertEquals(0, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test @@ -116,7 +140,9 @@ public void testProfilerInitWithProfiles() { state.put("config", helloWorldProfilerDef); StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(1, profiler.getConfig().getProfiles().size()); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test(expected = IllegalArgumentException.class) @@ -144,7 +170,9 @@ public void testProfilerInitWithNoGlobalConfig() { StandAloneProfiler profiler = executor.execute(expression, state, StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(1, profiler.getConfig().getProfiles().size()); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test @@ -158,19 +186,74 @@ public void testProfilerApply() { // apply a message to the profiler state.put("message", message); StandAloneProfiler result = run("PROFILER_APPLY(message, profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(1, profiler.getMessageCount()); + assertEquals(1, profiler.getRouteCount()); + } + + @Test + public void testProfilerApplyWithMultipleMessages() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + state.put("messages", messages); + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + + // validate assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(3, profiler.getMessageCount()); + assertEquals(3, profiler.getRouteCount()); + } + + @Test + public void testProfilerApplyWithEmptyList() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + state.put("messages", "[ ]"); + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test(expected = IllegalArgumentException.class) - public void testProfilerApplyNoArgs() { + public void testProfilerApplyWithNoArgs() { run("PROFILER_APPLY()", StandAloneProfiler.class); } @Test(expected = IllegalArgumentException.class) - public void testProfilerApplyInvalidArg() { + public void testProfilerApplyWithInvalidArg() { run("PROFILER_APPLY(undefined)", StandAloneProfiler.class); } + @Test(expected = IllegalArgumentException.class) + public void testProfilerApplyWithNullMessage() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // there is no 'messages' variable + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + } + @Test public void testProfilerFlush() { diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java index cf034c8233..6db70790fb 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java @@ -53,12 +53,28 @@ public class StandAloneProfiler { */ private MessageDistributor distributor; + /** + * Counts the number of messages that have been applied. + */ + private int messageCount; + + /** + * Counts the number of routes. + * + * If a message is not needed by any profiles, then there are 0 routes. + * If a message is needed by 1 profile then there is 1 route. + * If a message is needed by 2 profiles then there are 2 routes. + */ + private int routeCount; + public StandAloneProfiler(ProfilerConfig config, long periodDurationMillis, Context context) { this.context = context; this.config = config; this.router = new DefaultMessageRouter(context); // the period TTL does not matter in this context this.distributor = new DefaultMessageDistributor(periodDurationMillis, Long.MAX_VALUE); + this.messageCount = 0; + this.routeCount = 0; } /** @@ -72,6 +88,18 @@ public void apply(JSONObject message) throws ExecutionException { for(MessageRoute route : routes) { distributor.distribute(message, route, context); } + + routeCount += routes.size(); + messageCount += 1; + } + + @Override + public String toString() { + return "Profiler{" + + getProfileCount() + " profile(s), " + + getMessageCount() + " messages(s), " + + getRouteCount() + " route(s)" + + '}'; } /** @@ -85,4 +113,16 @@ public List flush() { public ProfilerConfig getConfig() { return config; } + + public int getProfileCount() { + return (config == null) ? 0: config.getProfiles().size(); + } + + public int getMessageCount() { + return messageCount; + } + + public int getRouteCount() { + return routeCount; + } } diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md index f27af59a51..b7aed55a72 100644 --- a/metron-analytics/metron-profiler/README.md +++ b/metron-analytics/metron-profiler/README.md @@ -281,6 +281,8 @@ In the following example, three values, the minimum, the maximum and the mean ar A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire. +The REPL can be a powerful for developing profiles. Read all about [Developing Profiles](../metron-profiler-client/#developing_profiles). + ## Configuring the Profiler The Profiler runs as an independent Storm topology. The configuration for the Profiler topology is stored in local filesystem at `$METRON_HOME/config/profiler.properties`. From 8ef18d31ab927acad7798c6c32252ca27c8e8b46 Mon Sep 17 00:00:00 2001 From: nickwallen Date: Thu, 31 Aug 2017 10:51:53 -0400 Subject: [PATCH 5/5] METRON-1138 Improve Error Message with Bad Profile Expression (nickwallen) closes apache/metron#721 --- .../profiler/DefaultProfileBuilder.java | 53 +++++++++++++------ .../profiler/DefaultProfileBuilderTest.java | 2 +- metron-analytics/metron-profiler/README.md | 1 + 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java index 1f352d0e0c..2e3416007b 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java @@ -27,9 +27,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.collections4.ListUtils; @@ -245,17 +247,28 @@ private Object execute(String expression, String expressionType) { * @param expressionType The type of expression; init, update, result. Provides additional context if expression execution fails. */ private void assign(Map expressions, Map transientState, String expressionType) { - try { - // execute each of the 'update' expressions - MapUtils.emptyIfNull(expressions) - .forEach((var, expr) -> executor.assign(var, expr, transientState)); + // for each expression... + for(Map.Entry entry : MapUtils.emptyIfNull(expressions).entrySet()) { + String var = entry.getKey(); + String expr = entry.getValue(); + + try { + // assign the result of the expression to the variable + executor.assign(var, expr, transientState); - } catch(ParseException e) { + } catch (Throwable e) { - // make it brilliantly clear that one of the 'update' expressions is bad - String msg = format("Bad '%s' expression: error=%s, profile=%s, entity=%s", expressionType, e.getMessage(), profileName, entity); - throw new ParseException(msg, e); + // in-scope variables = persistent state maintained by the profiler + the transient state + Set variablesInScope = new HashSet<>(); + variablesInScope.addAll(transientState.keySet()); + variablesInScope.addAll(executor.getState().keySet()); + + String msg = format("Bad '%s' expression: error='%s', expr='%s', profile='%s', entity='%s', variables-available='%s'", + expressionType, e.getMessage(), expr, profileName, entity, variablesInScope); + LOG.error(msg, e); + throw new ParseException(msg, e); + } } } @@ -269,14 +282,24 @@ private void assign(Map expressions, Map transie private List execute(List expressions, Map transientState, String expressionType) { List results = new ArrayList<>(); - try { - ListUtils.emptyIfNull(expressions) - .forEach((expr) -> results.add(executor.execute(expr, transientState, Object.class))); + for(String expr: ListUtils.emptyIfNull(expressions)) { + try { + // execute an expression + Object result = executor.execute(expr, transientState, Object.class); + results.add(result); + + } catch (Throwable e) { - } catch (Throwable e) { - String msg = format("Bad '%s' expression: error=%s, profile=%s, entity=%s", expressionType, e.getMessage(), profileName, entity); - LOG.error(msg, e); - throw new ParseException(msg, e); + // in-scope variables = persistent state maintained by the profiler + the transient state + Set variablesInScope = new HashSet<>(); + variablesInScope.addAll(transientState.keySet()); + variablesInScope.addAll(executor.getState().keySet()); + + String msg = format("Bad '%s' expression: error='%s', expr='%s', profile='%s', entity='%s', variables-available='%s'", + expressionType, e.getMessage(), expr, profileName, entity, variablesInScope); + LOG.error(msg, e); + throw new ParseException(msg, e); + } } return results; diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java index 71ef9827b8..d25b7ff13a 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java @@ -600,7 +600,7 @@ public void testBadResultExpression() throws Exception { * "init": { "x": "0" }, * "update": { "x": "x + 1" }, * "result": "x", - * "groupBy": ["2 / 0"] + * "groupBy": ["nonexistant"] * } */ @Multiline diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md index b7aed55a72..9b908f650c 100644 --- a/metron-analytics/metron-profiler/README.md +++ b/metron-analytics/metron-profiler/README.md @@ -200,6 +200,7 @@ A common use case would be grouping by day of week. This allows a contiguous sc ``` The expression can reference any of these variables. +* Any variable defined by the profile in its `init` or `update` expressions. * `profile` The name of the profile. * `entity` The name of the entity being profiled. * `start` The start time of the profile period in epoch milliseconds.