METRON-877 Extract core implementation and UDF support, create metron-stellar module

.gitignore

@@ -1,5 +1,6 @@
 *~
 target/
+*/target/
 *dependency-reduced-pom.xml
 .idea
 *.iml

dependencies_with_url.csv

@@ -19,6 +19,7 @@ com.esotericsoftware.minlog:minlog:jar:1.3.0:compile,New BSD License,http://code
 com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD License,http://code.google.com/p/minlog/
 com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD License,http://code.google.com/p/minlog/
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf
+com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB
 com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java

metron-analytics/metron-maas-service/pom.xml

@@ -37,6 +37,16 @@
       <artifactId>metron-maas-common</artifactId>
       <version>${project.parent.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.metron</groupId>
+      <artifactId>metron-common</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.metron</groupId>
+      <artifactId>stellar-common</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-common</artifactId>

metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java → metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java

@@ -15,17 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.metron.common.dsl.functions;
+package org.apache.metron.maas.functions;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.hadoop.security.authorize.Service;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.maas.config.Endpoint;
 import org.apache.metron.maas.config.MaaSConfig;

metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/MockDGAModel.java → metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.metron.common.stellar.maas;
+package org.apache.metron.maas.service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.ImmutableMap;

metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java → metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.metron.common.stellar.maas;
+package org.apache.metron.maas.service;
 
 import com.google.common.collect.ImmutableMap;
 import com.sun.jersey.server.impl.application.WebApplicationImpl;
@@ -28,7 +28,7 @@
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.ServiceInstanceBuilder;
 import org.apache.curator.x.discovery.ServiceType;
-import org.apache.metron.common.dsl.Context;
+import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.maas.config.Endpoint;
 import org.apache.metron.maas.config.MaaSConfig;
 import org.apache.metron.maas.config.ModelEndpoint;
@@ -42,7 +42,7 @@
 import java.util.Map;
 import java.util.logging.Level;
 
-import static org.apache.metron.common.utils.StellarProcessorUtils.run;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
 
 public class StellarMaaSIntegrationTest {
   private static Context context;

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java

@@ -19,10 +19,10 @@
  */
 package org.apache.metron.profiler.client.stellar;
 
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
 import org.apache.metron.profiler.ProfilePeriod;
 
 import java.util.List;

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java

@@ -22,10 +22,10 @@
 
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
 import org.apache.metron.hbase.HTableProvider;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.profiler.ProfilePeriod;

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java

@@ -20,7 +20,7 @@
 
 package org.apache.metron.profiler.client.stellar;
 
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
 import org.apache.metron.hbase.HTableProvider;
 
 import java.util.Map;

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java

@@ -20,21 +20,20 @@
 package org.apache.metron.profiler.client.stellar;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.lang.String.format;
-import static org.apache.metron.common.dsl.Context.Capabilities.GLOBAL_CONFIG;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
 
 public class Util {
   private static final Logger LOG = LoggerFactory.getLogger(Util.class);

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java

@@ -21,11 +21,11 @@
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
 import org.apache.metron.profiler.ProfilePeriod;
 import org.apache.metron.profiler.client.window.Window;
 import org.apache.metron.profiler.client.window.WindowProcessor;

metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java

@@ -21,11 +21,11 @@
 
 import org.antlr.v4.runtime.*;
 import org.antlr.v4.runtime.tree.ParseTree;
-import org.apache.metron.common.dsl.ErrorListener;
-import org.apache.metron.common.dsl.GrammarUtils;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Token;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.ErrorListener;
+import org.apache.metron.stellar.dsl.GrammarUtils;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Token;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
 import org.apache.metron.profiler.client.window.generated.WindowBaseListener;
 import org.apache.metron.profiler.client.window.generated.WindowLexer;
 import org.apache.metron.profiler.client.window.generated.WindowParser;

metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java

@@ -22,10 +22,9 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
-import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.profiler.ProfileMeasurement;
 import org.apache.metron.profiler.client.stellar.FixedLookback;
@@ -34,8 +33,8 @@
 import org.apache.metron.profiler.hbase.RowKeyBuilder;
 import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
 import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
 import org.apache.metron.test.mock.MockHTable;
 import org.junit.Assert;
 import org.junit.Before;
@@ -62,7 +61,7 @@ public class GetProfileTest {
   private static final int saltDivisor = 1000;
   private static final String tableName = "profiler";
   private static final String columnFamily = "P";
-  private StellarExecutor executor;
+  private StellarStatefulExecutor executor;
   private Map<String, Object> state;
   private ProfileWriter profileWriter;
   // different values of period and salt divisor, used to test config_overrides feature
@@ -119,7 +118,7 @@ public void setup() {
     }};
 
     // create the stellar execution environment
-    executor = new DefaultStellarExecutor(
+    executor = new DefaultStellarStatefulExecutor(
             new SimpleFunctionResolver()
                     .withClass(GetProfile.class)
                     .withClass(FixedLookback.class),
@@ -140,9 +139,9 @@ public void setup() {
    * original context values in the PROFILE_GET config_overrides argument gets all expected results.
    *
    * @return context2 - The profiler client configuration context created by this method.
-   *    The context2 values are also set in the configuration of the StellarExecutor
+   *    The context2 values are also set in the configuration of the StellarStatefulExecutor
    *    stored in the global variable 'executor'.  However, there is no API for querying the
-   *    context values from a StellarExecutor, so we output the context2 Context object itself,
+   *    context values from a StellarStatefulExecutor, so we output the context2 Context object itself,
    *    for validation purposes (so that its values can be validated as being significantly
    *    different from the setup() settings).
    */
@@ -165,7 +164,7 @@ private Context setup2() {
             .build();
 
     // create the stellar execution environment
-    executor = new DefaultStellarExecutor(
+    executor = new DefaultStellarStatefulExecutor(
             new SimpleFunctionResolver()
                     .withClass(GetProfile.class)
                     .withClass(FixedLookback.class),

metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java

@@ -25,8 +25,8 @@
 import org.apache.metron.profiler.hbase.RowKeyBuilder;
 import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
 import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
 import org.apache.metron.test.mock.MockHTable;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -58,15 +58,15 @@ public class HBaseProfilerClientTest {
   private static final int periodsPerHour = 4;
 
   private HBaseProfilerClient client;
+  private StellarStatefulExecutor executor;
   private MockHTable table;
-  private StellarExecutor executor;
   private ProfileWriter profileWriter;
 
   @Before
   public void setup() throws Exception {
 
     table = new MockHTable(tableName, columnFamily);
-    executor = new DefaultStellarExecutor();
+    executor = new DefaultStellarStatefulExecutor();
 
     // used to write values to be read during testing
     RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder();

metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java

@@ -21,11 +21,11 @@
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang3.Range;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
-import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.metron.stellar.common.StellarProcessor;
 import org.apache.metron.profiler.ProfilePeriod;
 import org.apache.metron.profiler.client.window.WindowProcessor;
 import org.junit.Assert;

metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java

@@ -293,27 +293,27 @@ public void testDateDaySpecifier() throws ParseException {
     }
   }
 
-  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
   public void testWithInvalidDaySpecifier() throws ParseException {
     WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding hoolidays:us");
   }
 
-  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
   public void testWithInvalidTimeUnit() throws ParseException {
     WindowProcessor.process("30 minute window every 24 months from 14 days ago");
   }
 
-  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
   public void testWithInvalidWindowUnit() throws ParseException {
     WindowProcessor.process("30 minuete window every 24 hours from 14 days ago");
   }
 
-  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
   public void testWithInvalidTimeNumber() throws ParseException {
     WindowProcessor.process("30p minute window every 24 hours from 14 days ago");
   }
 
-  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
   public void testInvalidDaySpecifier() throws ParseException {
     WindowProcessor.process("30 minute window every 14 hours from 14 days ago including date");
   }

metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java

@@ -26,22 +26,20 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
-import org.apache.metron.common.configuration.profiler.ProfileResult;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.apache.metron.profiler.clock.Clock;
 import org.apache.metron.profiler.clock.WallClock;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -81,7 +79,7 @@ public class ProfileBuilder implements Serializable {
   /**
    * Executes Stellar code and maintains state across multiple invocations.
    */
-  private StellarExecutor executor;
+  private StellarStatefulExecutor executor;
 
   /**
    * Has the profile been initialized?
@@ -114,7 +112,7 @@ private ProfileBuilder(ProfileConfig definition,
     this.entity = entity;
     this.clock = clock;
     this.periodDurationMillis = periodDurationMillis;
-    this.executor = new DefaultStellarExecutor();
+    this.executor = new DefaultStellarStatefulExecutor();
     Context context = new Context.Builder()
             .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
             .with(Context.Capabilities.GLOBAL_CONFIG, () -> global)

metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java

@@ -23,7 +23,6 @@
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.profiler.clock.Clock;
 import org.apache.metron.profiler.clock.FixedClock;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -32,7 +31,7 @@
 
 import java.util.concurrent.TimeUnit;
 
-import static org.apache.metron.common.utils.ConversionUtils.convert;
+import static org.apache.metron.stellar.common.utils.ConversionUtils.convert;
 import static org.junit.Assert.assertEquals;
 
 /**