From 5cb3a9354efbba6c8f9b85382848152839407d68 Mon Sep 17 00:00:00 2001 From: Mohan Venkateshaiah Date: Sat, 24 Feb 2018 01:29:07 +0530 Subject: [PATCH 1/4] Modify the MIN, MAX Stellar methods to take a stats or list object and return min/max. --- .../metron/statistics/StatisticsProvider.java | 6 +- .../metron/stellar/dsl/functions/Ordinal.java | 24 ++++++++ .../dsl/functions/OrdinalFunctions.java | 28 ++++++--- .../dsl/functions/OrdinalFunctionsTest.java | 57 ++++++++++++++++++- 4 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java index 860aa4e6a9..e737484d00 100644 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java @@ -20,14 +20,14 @@ package org.apache.metron.statistics; +import org.apache.metron.stellar.dsl.functions.Ordinal; + /** * Provides statistical functions. */ -public interface StatisticsProvider { +public interface StatisticsProvider extends Ordinal{ void addValue(double value); long getCount(); - double getMin(); - double getMax(); double getMean(); double getSum(); double getVariance(); diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java new file mode 100644 index 0000000000..87bf5560c4 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.dsl.functions; + +public interface Ordinal { + + double getMin(); + double getMax(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java index 49e93690e8..2a02ff9d05 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java @@ -22,12 +22,10 @@ import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; - -import java.util.Collections; import java.util.List; import java.util.function.BiFunction; -import java.util.function.Predicate; -import java.util.stream.Collectors; +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; + public class OrdinalFunctions { @@ -37,10 +35,11 @@ public class OrdinalFunctions { * Return the maximum value of a list of input values in a Stellar list */ @Stellar(name = "MAX" - , description = "Returns the maximum value of a list of input values" - , params = {"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" + + , description = "Returns the maximum value of a list of input values or from a statistics object" + , params = {"stats - The Stellar statistics object" + ,"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" + " Multi type numeric comparisons are supported: MAX([10,15L,15.3]) would return 15.3, but MAX(['23',25]) will fail and return null as strings and numbers can't be compared."} - , returns = "The maximum value in the list, or null if the list is empty or the input values were not comparable.") + , returns = "The maximum value in the list or from stats, or null if the list is empty or the input values were not comparable.") public static class Max extends BaseStellarFunction { @Override @@ -48,6 +47,11 @@ public Object apply(List args) { if (args.size() < 1 || args.get(0) == null) { throw new IllegalStateException("MAX function requires at least a Stellar list of values"); } + Object firstArg = args.get(0); + if(firstArg instanceof Ordinal) { + Ordinal stats = convert(firstArg, Ordinal.class); + return stats.getMax(); + } Iterable list = (Iterable) args.get(0); return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); } @@ -60,15 +64,21 @@ public Object apply(List args) { */ @Stellar(name = "MIN" , description = "Returns the minimum value of a list of input values" - , params = {"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" + + , params = {"stats - The Stellar statistics object" + ,"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" + " Multi type numeric comparisons are supported: MIN([10,15L,15.3]) would return 10, but MIN(['23',25]) will fail and return null as strings and numbers can't be compared."} - , returns = "The minimum value in the list, or null if the list is empty or the input values were not comparable.") + , returns = "The minimum value in the list or from stats, or null if the list is empty or the input values were not comparable.") public static class Min extends BaseStellarFunction { @Override public Object apply(List args) { if (args.size() < 1 || args.get(0) == null) { throw new IllegalStateException("MIN function requires at least a Stellar list of values"); } + Object firstArg = args.get(0); + if(firstArg instanceof Ordinal) { + Ordinal stats = convert(firstArg, Ordinal.class); + return stats.getMin(); + } Iterable list = (Iterable) args.get(0); return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java index 5e06fdd40c..39978e252f 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java @@ -20,7 +20,8 @@ import com.google.common.collect.ImmutableMap; -import org.adrianwalker.multilinestring.Multiline; +import org.apache.commons.math.random.GaussianRandomGenerator; +import org.apache.commons.math.random.MersenneTwister; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; @@ -28,7 +29,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import java.util.*; @@ -220,6 +220,59 @@ public TestObject(String arg) { } } + + @Test + public void testMaxOfStats() throws Exception { + + GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L)); + List sample = new ArrayList(); + for(int j = 0;j < 100;++j) { + double s = gaussian.nextNormalizedDouble(); + sample.add(s); + } + Ordinal provider = new Ordinal() { + @Override + public double getMin() { + return 10; + } + + @Override + public double getMax() { + return 100; + } + }; + + Object res = run("MAX(input_list)", ImmutableMap.of("input_list", provider)); + Assert.assertNotNull(res); + Assert.assertTrue(res.equals(100.0d)); + } + + @Test + public void testMinOfStats() throws Exception { + + GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L)); + List sample = new ArrayList(); + for(int j = 0;j < 100;++j) { + double s = gaussian.nextNormalizedDouble(); + sample.add(s); + } + Ordinal provider = new Ordinal() { + @Override + public double getMin() { + return 10; + } + + @Override + public double getMax() { + return 100; + } + }; + + Object res = run("MIN(input_list)", ImmutableMap.of("input_list", provider)); + Assert.assertNotNull(res); + Assert.assertTrue(res.equals(10.0d)); + } + public Object run(String rule, Map variables) throws Exception { StellarProcessor processor = new StellarProcessor(); return processor.parse(rule, new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), context); From 1fc1459ac45ccd21ff1975d67adf280d1f2bb171 Mon Sep 17 00:00:00 2001 From: Mohan Venkateshaiah Date: Sat, 24 Feb 2018 01:44:35 +0530 Subject: [PATCH 2/4] removed the unused code. --- .../dsl/functions/OrdinalFunctionsTest.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java index 39978e252f..e405c7f9b5 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java @@ -20,8 +20,6 @@ import com.google.common.collect.ImmutableMap; -import org.apache.commons.math.random.GaussianRandomGenerator; -import org.apache.commons.math.random.MersenneTwister; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; @@ -223,13 +221,6 @@ public TestObject(String arg) { @Test public void testMaxOfStats() throws Exception { - - GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L)); - List sample = new ArrayList(); - for(int j = 0;j < 100;++j) { - double s = gaussian.nextNormalizedDouble(); - sample.add(s); - } Ordinal provider = new Ordinal() { @Override public double getMin() { @@ -249,13 +240,6 @@ public double getMax() { @Test public void testMinOfStats() throws Exception { - - GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L)); - List sample = new ArrayList(); - for(int j = 0;j < 100;++j) { - double s = gaussian.nextNormalizedDouble(); - sample.add(s); - } Ordinal provider = new Ordinal() { @Override public double getMin() { From 31ae4df19f061ce0f6544080ac63589a2ba1bc92 Mon Sep 17 00:00:00 2001 From: Mohan Venkateshaiah Date: Mon, 26 Feb 2018 23:38:39 +0530 Subject: [PATCH 3/4] incorporated review comments. --- .../metron/stellar/dsl/functions/Ordinal.java | 12 +++++++++++ .../dsl/functions/OrdinalFunctions.java | 20 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java index 87bf5560c4..d3bd9ce131 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java @@ -17,8 +17,20 @@ */ package org.apache.metron.stellar.dsl.functions; +/** + * Interface that provides the statistical function get max and min from the implementing object. + */ public interface Ordinal { + /** + * get the min value + * @return min value + */ double getMin(); + + /** + * get the max value + * @return max value + */ double getMax(); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java index 2a02ff9d05..896b19aada 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java @@ -45,15 +45,19 @@ public static class Max extends BaseStellarFunction { @Override public Object apply(List args) { if (args.size() < 1 || args.get(0) == null) { - throw new IllegalStateException("MAX function requires at least a Stellar list of values"); + throw new IllegalStateException("MAX function requires at least one argument"); } Object firstArg = args.get(0); if(firstArg instanceof Ordinal) { Ordinal stats = convert(firstArg, Ordinal.class); return stats.getMax(); + } else if (firstArg instanceof Iterable) { + Iterable list = (Iterable) args.get(0); + return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); + } else { + throw new IllegalStateException("MAX function expects either Stellar statistics object or Stellar list of values "); } - Iterable list = (Iterable) args.get(0); - return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); + } } @@ -72,16 +76,18 @@ public static class Min extends BaseStellarFunction { @Override public Object apply(List args) { if (args.size() < 1 || args.get(0) == null) { - throw new IllegalStateException("MIN function requires at least a Stellar list of values"); + throw new IllegalStateException("MIN function requires at least one argument"); } Object firstArg = args.get(0); if(firstArg instanceof Ordinal) { Ordinal stats = convert(firstArg, Ordinal.class); return stats.getMin(); + } else if (firstArg instanceof Iterable){ + Iterable list = (Iterable) args.get(0); + return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); + } else { + throw new IllegalStateException("MIN function expects either Stellar statistics object or Stellar list of values "); } - Iterable list = (Iterable) args.get(0); - return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); - } } From 3267030432ed57e66b4092a1534a3edaabd1595b Mon Sep 17 00:00:00 2001 From: Mohan Venkateshaiah Date: Mon, 16 Apr 2018 17:38:27 +0530 Subject: [PATCH 4/4] Restrict MAX and MIN accept either A StatisticsProvider object or A list of comparables. --- .../metron/stellar/dsl/functions/OrdinalFunctions.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java index 896b19aada..6ac9ff515c 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java @@ -52,10 +52,10 @@ public Object apply(List args) { Ordinal stats = convert(firstArg, Ordinal.class); return stats.getMax(); } else if (firstArg instanceof Iterable) { - Iterable list = (Iterable) args.get(0); + Iterable list = (Iterable) args.get(0); return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); } else { - throw new IllegalStateException("MAX function expects either Stellar statistics object or Stellar list of values "); + throw new IllegalStateException("MAX function expects either 'a StatisticsProvider object' or 'Stellar list of values'"); } } @@ -86,7 +86,7 @@ public Object apply(List args) { Iterable list = (Iterable) args.get(0); return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); } else { - throw new IllegalStateException("MIN function expects either Stellar statistics object or Stellar list of values "); + throw new IllegalStateException("MIN function expects either 'a StatisticsProvider object' or 'Stellar list of values' "); } } }