METRON-690: Create a DSL-based timestamp lookup for profiler to enable sparse windows

dependencies_with_url.csv

@@ -221,4 +221,5 @@ org.yaml:snakeyaml:jar:1.15:compile,Apache License Version 2.0,http://www.snakey
 ring-cors:ring-cors:jar:0.1.5:compile,Eclipse Public License 1.0,https://github.com/r0man/ring-cors
 xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j
 xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/
-
+de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html
+org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html

metron-analytics/metron-profiler-client/pom.xml

@@ -58,6 +58,11 @@
             </exclusions>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>de.jollyday</groupId>
+            <artifactId>jollyday</artifactId>
+            <version>0.5.2</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
             <artifactId>metron-profiler-common</artifactId>
@@ -236,6 +241,12 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr4-runtime</artifactId>
+            <version>${global_antlr_version}</version>
+        </dependency>
+
     </dependencies>
     <reporting>
         <plugins>
@@ -265,6 +276,22 @@
     </reporting>
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-maven-plugin</artifactId>
+                <version>${global_antlr_version}</version>
+                <configuration>
+                  <outputDirectory>${basedir}/src/main/java</outputDirectory>
+                </configuration>
+                <executions>
+                  <execution>
+                    <goals>
+                      <goal>antlr4</goal>
+                    </goals>
+                  </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>

metron-analytics/metron-profiler-client/src/main/antlr4/org/apache/metron/profiler/client/window/generated/Window.g4

@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grammar Window;
+
+@header {
+//CHECKSTYLE:OFF
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+}
+COMMA : ',';
+COLON : ':';
+
+WINDOW : 'window' | 'windows';
+
+INCLUDE : 'include' | 'INCLUDE' | 'includes' | 'INCLUDES' | 'including' | 'INCLUDING';
+EXCLUDE : 'exclude' | 'EXCLUDE' | 'excludes' | 'EXCLUDES' | 'excluding' | 'EXCLUDING';
+
+NOW : 'NOW' | 'now';
+
+FROM : 'FROM' | 'from' | 'STARTING FROM' | 'starting from';
+EVERY : 'EVERY' | 'every' | 'FOR EVERY' | 'for every';
+TO : 'TO' | 'to' | 'until' | 'UNTIL';
+AGO : 'AGO' | 'ago';
+
+NUMBER : FIRST_DIGIT DIGIT*;
+IDENTIFIER : [:][a-zA-Z0-9][a-zA-Z0-9_\.\-/]*;
+
+DAY_SPECIFIER : MONDAY | TUESDAY | WEDNESDAY | THURSDAY
+                       | FRIDAY | SATURDAY | SUNDAY
+                       | CURRENT_DAY_OF_WEEK
+                       | WEEKEND | WEEKDAY | HOLIDAYS
+                       | DATE
+                       ;
+
+TIME_UNIT : SECOND_UNIT | MINUTE_UNIT | HOUR_UNIT | DAY_UNIT ;
+
+WS : [ \r\t\u000C\n]+ -> skip;
+
+fragment SECOND_UNIT : 'SECOND' | 'second' | 'seconds' | 'SECONDS' | 'second(s)' | 'SECOND(S)';
+fragment MINUTE_UNIT : 'MINUTE' | 'minute' | 'minutes' | 'MINUTES' | 'minute(s)' | 'MINUTE(S)';
+fragment HOUR_UNIT : 'HOUR' | 'hour' | 'hours' | 'HOURS' | 'hour(s)' | 'HOUR(S)';
+fragment DAY_UNIT : 'DAY' | 'day' | 'days' | 'DAYS' | 'day(s)' | 'DAY(S)';
+fragment MONDAY : 'MONDAY' | 'monday' | 'MONDAYS' | 'mondays';
+fragment TUESDAY : 'TUESDAY' | 'tuesday' | 'TUESDAYS' | 'tuesdays';
+fragment WEDNESDAY : 'WEDNESDAY' | 'wednesday' | 'WEDNESDAYS' | 'wednesdays';
+fragment THURSDAY : 'THURSDAY' | 'thursday' | 'THURSDAYS' | 'thursdays';
+fragment FRIDAY : 'FRIDAY' | 'friday' | 'FRIDAYS' | 'fridays';
+fragment SATURDAY: 'SATURDAY' | 'saturday' | 'SATURDAYS' | 'saturdays';
+fragment SUNDAY : 'SUNDAY' | 'sunday' | 'SUNDAYS' | 'sundays';
+fragment CURRENT_DAY_OF_WEEK: 'this day of week' | 'THIS DAY OF WEEK' | 'this day of the week' | 'THIS DAY OF THE WEEK'
+                            | 'current day of week' | 'CURRENT DAY OF WEEK'
+                            | 'current day of the week' | 'CURRENT DAY OF THE WEEK';
+fragment WEEKEND : 'weekend' | 'WEEKEND' | 'weekends' | 'WEEKENDS';
+fragment WEEKDAY: 'weekday' | 'WEEKDAY' | 'weekdays' | 'WEEKDAYS';
+fragment HOLIDAYS: 'holiday' | 'HOLIDAY' | 'holidays' | 'HOLIDAYS';
+fragment DATE: 'date' | 'DATE';
+
+fragment DIGIT : '0'..'9';
+fragment FIRST_DIGIT : '1'..'9';
+
+window : window_expression EOF;
+
+window_expression : window_width including_specifier? excluding_specifier? #NonRepeatingWindow
+                  | window_width skip_distance duration including_specifier? excluding_specifier? #RepeatingWindow
+                  | duration #DenseWindow
+                  ;
+
+excluding_specifier : EXCLUDE specifier_list
+                    ;
+including_specifier : INCLUDE specifier_list
+                    ;
+
+specifier : day_specifier
+          | day_specifier specifier_arg_list
+          ;
+
+specifier_arg_list : identifier
+                   | identifier specifier_arg_list
+                    ;
+
+day_specifier : DAY_SPECIFIER ;
+
+identifier : NUMBER | IDENTIFIER
+          ;
+
+specifier_list : specifier
+               | specifier_list COMMA specifier
+               ;
+
+duration : FROM time_interval AGO? TO time_interval AGO? #FromToDuration
+         | FROM time_interval AGO? #FromDuration
+         ;
+
+skip_distance : EVERY time_interval #SkipDistance
+              ;
+
+window_width : time_interval WINDOW? #WindowWidth
+          ;
+
+time_interval : time_amount time_unit #TimeInterval
+              ;
+
+time_amount : NUMBER #TimeAmount
+            ;
+
+time_unit : TIME_UNIT #TimeUnit
+            ;

metron-analytics/metron-profiler-client/src/main/java/Window.tokens

@@ -0,0 +1,17 @@
+COMMA=1
+COLON=2
+WINDOW=3
+INCLUDE=4
+EXCLUDE=5
+NOW=6
+FROM=7
+EVERY=8
+TO=9
+AGO=10
+NUMBER=11
+IDENTIFIER=12
+DAY_SPECIFIER=13
+TIME_UNIT=14
+WS=15
+','=1
+':'=2

metron-analytics/metron-profiler-client/src/main/java/WindowLexer.tokens

@@ -0,0 +1,17 @@
+COMMA=1
+COLON=2
+WINDOW=3
+INCLUDE=4
+EXCLUDE=5
+NOW=6
+FROM=7
+EVERY=8
+TO=9
+AGO=10
+NUMBER=11
+IDENTIFIER=12
+DAY_SPECIFIER=13
+TIME_UNIT=14
+WS=15
+','=1
+':'=2

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/IntervalPredicate.java

@@ -0,0 +1,79 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.client.stellar;
+
+import org.apache.commons.lang3.Range;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class IntervalPredicate<T> implements Predicate<T> {
+  private final List<Range<Long>> intervals;
+  private final Function<T, Long> timestampTransformer;
+
+  public static final class Identity extends IntervalPredicate<Long> {
+
+    public Identity(List<Range<Long>> intervals) {
+      super(x -> x, intervals, Long.class);
+    }
+  }
+
+  public IntervalPredicate(Function<T, Long> timestampTransformer, List<Range<Long>> intervals, Class<T> clazz) {
+    this.intervals = intervals;
+    this.timestampTransformer = timestampTransformer;
+  }
+
+  private boolean containsInclusive(Range<Long> interval, long ts) {
+    return interval.contains(ts) || interval.getMaximum() == ts;
+  }
+
+
+  public static final Comparator<Range<Long>> INTERVAL_COMPARATOR = (o1, o2) -> {
+      if(o1.getMinimum() == o2.getMinimum() && o1.getMaximum() == o2.getMaximum()) {
+        return 0;
+      }
+      else {
+        int ret = Long.compare(o1.getMinimum(), o2.getMinimum());
+        if(ret == 0) {
+          return Long.compare(o1.getMaximum(), o2.getMaximum());
+        }
+        else {
+          return ret;
+        }
+      }
+  };
+
+  @Override
+  public boolean test(T x) {
+    long ts = timestampTransformer.apply(x);
+    int pos = Collections.binarySearch(intervals, Range.is(ts), INTERVAL_COMPARATOR);
+    if(pos < 0) {
+      pos = -pos - 1;
+    }
+
+    Optional<Range<Long>> right = pos >= 0 && pos < intervals.size()?Optional.of(intervals.get(pos)):Optional.empty();
+    Optional<Range<Long>> left = pos - 1 >= 0 && pos - 1 < intervals.size()?Optional.of(intervals.get(pos - 1)):Optional.empty();
+    return (right.isPresent()?containsInclusive(right.get(),ts):false) || (left.isPresent()?containsInclusive(left.get(),ts):false);
+  }
+}