METRON-1107 add support for handling epoch dates in seconds or milliseconds

metron-stellar/stellar-common/README.md

@@ -289,19 +289,19 @@ In the core language functions, we support basic functional programming primitiv
 ### `DAY_OF_MONTH`
   * Description: The numbered day within the month.  The first day within the month has a value of 1.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The numbered day within the month.
 
 ### `DAY_OF_WEEK`
   * Description: The numbered day within the week.  The first day of the week, Sunday, has a value of 1.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The numbered day within the week.
 
 ### `DAY_OF_YEAR`
   * Description: The day number within the year.  The first day of the year has value of 1.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The day number within the year.
 
 ### `DECODE`
@@ -608,7 +608,7 @@ In the core language functions, we support basic functional programming primitiv
 ### `MONTH`
   * Description: The number representing the month.  The first month, January, has a value of 0.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The current month (0-based).
 
 ### `PREPEND_IF_MISSING`
@@ -813,19 +813,19 @@ In the core language functions, we support basic functional programming primitiv
 ### `WEEK_OF_MONTH`
   * Description: The numbered week within the month.  The first week within the month has a value of 1.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The numbered week within the month.
 
 ### `WEEK_OF_YEAR`
   * Description: The numbered week within the year.  The first week in the year has a value of 1.
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The numbered week within the year.
 
 ### `YEAR`
   * Description: The number representing the year. 
   * Input:
-    * dateTime - The datetime as a long representing the milliseconds since unix epoch
+    * dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch
   * Returns: The current year
 
 ### `ZIP`

metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/EpochUtils.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.common.utils;
+
+/**
+ * Utility functions for working with numbers that represent different
+ * formats of EPOCH time.
+ */
+public class EpochUtils {
+
+  /**
+   * Ensures returns the passed value as milliseconds from EPOCH if the value is in seconds.
+   * This is done by looking at the number of digits.
+   * If there are 10, then the value is concidered to be in seconds and will by
+   * muliplited by 1000.
+   * If there not 10, then the original value will be returned.
+   *
+   *
+   * </p>
+   * @param canidate The Long value to concider
+   * @return A Long value
+   */
+  public static Long ensureEpochMillis(Long canidate) {
+    int length = (int)Math.floor(Math.log10(canidate) + 1);
+    return length == 10 ? canidate * 1000 : canidate;
+  }
+
+}

metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java

@@ -21,6 +21,7 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import org.apache.metron.stellar.common.utils.EpochUtils;
 import org.apache.metron.stellar.dsl.BaseStellarFunction;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.common.utils.ConversionUtils;
@@ -174,7 +175,7 @@ private static <T> T getOrDefault(List<Object> args, int position, Class<T> claz
    */
   @Stellar( name="DAY_OF_WEEK"
           , description="The numbered day within the week.  The first day of the week, Sunday, has a value of 1."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The numbered day within the week.")
   public static class DayOfWeek extends BaseStellarFunction {
@@ -187,6 +188,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -202,7 +205,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="DAY_OF_MONTH"
           , description="The numbered day within the month.  The first day within the month has a value of 1."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The numbered day within the month.")
   public static class DayOfMonth extends BaseStellarFunction {
@@ -215,6 +218,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -230,7 +235,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="WEEK_OF_MONTH"
           , description="The numbered week within the month.  The first week within the month has a value of 1."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The numbered week within the month.")
   public static class WeekOfMonth extends BaseStellarFunction {
@@ -243,6 +248,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -258,7 +265,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="WEEK_OF_YEAR"
           , description="The numbered week within the year.  The first week in the year has a value of 1."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The numbered week within the year.")
   public static class WeekOfYear extends BaseStellarFunction {
@@ -271,6 +278,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -286,7 +295,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="MONTH"
           , description="The number representing the month.  The first month, January, has a value of 0."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The current month (0-based).")
   public static class MonthOfYear extends BaseStellarFunction {
@@ -299,6 +308,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -314,7 +325,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="YEAR"
           , description="The number representing the year. "
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The current year"
           )
@@ -328,6 +339,8 @@ public Object apply(List<Object> args) {
         return null;  // invalid argument
       }
 
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
+
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);
@@ -343,7 +356,7 @@ public Object apply(List<Object> args) {
    */
   @Stellar( name="DAY_OF_YEAR"
           , description="The day number within the year.  The first day of the year has value of 1."
-          , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch"
+          , params = { "dateTime - The datetime as a long representing the milliseconds or seconds since unix epoch"
                      }
           , returns = "The day number within the year."
           )
@@ -356,7 +369,7 @@ public Object apply(List<Object> args) {
       if(epochMillis == null) {
         return null;  // invalid argument
       }
-
+      epochMillis = EpochUtils.ensureEpochMillis(epochMillis);
       // create a calendar
       Calendar calendar = Calendar.getInstance();
       calendar.setTimeInMillis(epochMillis);

metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/EpochUtilsTest.java

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.common.utils;
+
+import static org.junit.Assert.*;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class EpochUtilsTest {
+
+  @Test
+  public void testSecondsToMillis() throws Exception {
+    Long seconds = 1472131630L;
+    Long value = EpochUtils.ensureEpochMillis(seconds);
+    Assert.assertTrue(seconds.longValue() * 1000 == EpochUtils.ensureEpochMillis(seconds).longValue());
+  }
+
+  @Test
+  public void testMillisStaysMillis() throws Exception {
+    Long millis = 1472131630748L;
+    Long value = EpochUtils.ensureEpochMillis(millis);
+    Assert.assertTrue(millis.longValue() == EpochUtils.ensureEpochMillis(millis).longValue());
+  }
+
+  @Test
+  public void testOtherLengthToMillis() throws Exception {
+    Long meh = 1L;
+    Assert.assertTrue(meh.longValue() == EpochUtils.ensureEpochMillis(meh).longValue());
+  }
+
+
+
+}

metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java

@@ -57,10 +57,12 @@ private Object run(String expr) {
    * Thu Aug 25 2016 09:27:10 EST
    */
   private long AUG2016 = 1472131630748L;
+  private long AUG2016_SECONDS = 1472131630L;
 
   @Before
   public void setup() {
     variables.put("epoch", AUG2016);
+    variables.put("epoch_seconds", AUG2016_SECONDS);
     calendar = Calendar.getInstance();
   }
 
@@ -70,6 +72,12 @@ public void testDayOfWeek() {
     assertEquals(Calendar.THURSDAY, result);
   }
 
+  @Test
+  public void testDayOfWeekSeconds() {
+    Object result = run("DAY_OF_WEEK(epoch_seconds)");
+    assertEquals(Calendar.THURSDAY, result);
+  }
+
   /**
    * If no argument, then return the current day of week.
    */
@@ -116,6 +124,12 @@ public void testMonth() {
     assertEquals(Calendar.AUGUST, result);
   }
 
+  @Test
+  public void testMonthSeconds() {
+    Object result = run("MONTH(epoch_seconds)");
+    assertEquals(Calendar.AUGUST, result);
+  }
+
   /**
    * If no argument, then return the current month.
    */
@@ -139,6 +153,12 @@ public void testYear() {
     assertEquals(2016, result);
   }
 
+  @Test
+  public void testYearSeconds() {
+    Object result = run("YEAR(epoch_seconds)");
+    assertEquals(2016, result);
+  }
+
   /**
    * If no argument, then return the current year.
    */
@@ -162,6 +182,12 @@ public void testDayOfMonth() {
     assertEquals(25, result);
   }
 
+  @Test
+  public void testDayOfMonthSeconds() {
+    Object result = run("DAY_OF_MONTH(epoch_seconds)");
+    assertEquals(25, result);
+  }
+
   /**
    * If no argument, then return the current day of month.
    */
@@ -185,6 +211,12 @@ public void testWeekOfYear() {
     assertEquals(35, result);
   }
 
+  @Test
+  public void testWeekOfYearSeconds() {
+    Object result = run("WEEK_OF_YEAR(epoch_seconds)");
+    assertEquals(35, result);
+  }
+
   /**
    * If no argument, then return the current week of year.
    */
@@ -208,6 +240,12 @@ public void testDayOfYear() {
     assertEquals(238, result);
   }
 
+  @Test
+  public void testDayOfYearSeconds() {
+    Object result = run("DAY_OF_YEAR(epoch_seconds)");
+    assertEquals(238, result);
+  }
+
   /**
    * If no argument, then return the current day of year.
    */