From 3888a19989d4fd58c54bc9dd84d7a5ac8ddbc7ee Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 15 Aug 2016 08:42:57 -0400 Subject: [PATCH 1/3] METRON-366: Add MODEL_APPLY to Stellar --- .../apache/metron/maas/config/Endpoint.java | 72 ++ .../metron/maas/config/ModelEndpoint.java | 36 +- .../maas/discovery/ServiceDiscoverer.java | 73 +- .../apache/metron/maas/util/ConfigUtil.java | 19 +- .../org/apache/metron/maas/util/RESTUtil.java | 77 ++ .../ServiceDiscoveryIntegrationTest.java | 8 +- .../metron/maas/service/runner/Runner.java | 21 +- .../metron/maas/submit/ModelSubmission.java | 2 +- .../maas/service/MaasIntegrationTest.java | 4 +- .../src/test/resources/maas/dummy_rest.sh | 2 +- metron-platform/metron-common/pom.xml | 5 + .../common/stellar/generated/Stellar.g4 | 45 +- .../src/main/java/Stellar.tokens | 63 +- .../src/main/java/StellarLexer.tokens | 63 +- .../metron/common/bolt/ConfiguredBolt.java | 5 + .../configuration/FieldTransformer.java | 11 +- .../common/configuration/FieldValidator.java | 5 +- .../common/dsl/BaseStellarFunction.java | 35 + .../org/apache/metron/common/dsl/Context.java | 85 ++ .../metron/common/dsl/FunctionResolver.java | 24 + .../metron/common/dsl/StellarFunction.java | 25 + .../metron/common/dsl/StellarFunctions.java | 138 +- .../dsl/functions/ConversionFunctions.java | 37 + .../dsl/functions/DataStructureFunctions.java | 46 + .../common/dsl/functions/DateFunctions.java | 3 +- .../common/dsl/functions/MaaSFunctions.java | 259 ++++ .../common/dsl/functions/MapFunctions.java | 19 +- .../dsl/functions/NetworkFunctions.java | 41 +- .../common/dsl/functions/StringFunctions.java | 80 +- .../transformation/FieldTransformation.java | 3 + .../IPProtocolTransformation.java | 12 +- .../transformation/RemoveTransformation.java | 10 +- .../SimpleFieldTransformation.java | 2 + .../transformation/StellarTransformation.java | 5 +- .../field/validation/FieldValidation.java | 3 + .../field/validation/QueryValidation.java | 5 +- .../field/validation/SimpleValidation.java | 3 + .../validation/network/IPValidation.java | 2 + .../validation/primitive/DateValidation.java | 2 + .../validation/primitive/RegexValidation.java | 2 + .../common/stellar/BaseStellarProcessor.java | 17 +- .../common/stellar/StellarCompiler.java | 75 +- .../stellar/StellarPredicateProcessor.java | 14 +- .../generated/StellarBaseListener.java | 120 +- .../stellar/generated/StellarLexer.java | 210 +-- .../stellar/generated/StellarListener.java | 164 ++- .../stellar/generated/StellarParser.java | 1122 +++++++++++------ .../FieldTransformationTest.java | 5 +- .../RemoveTransformationTest.java | 9 +- .../StellarTransformationTest.java | 11 +- .../field/validation/BaseValidationTest.java | 5 +- .../metron/common/stellar/StellarTest.java | 115 +- .../common/stellar/maas/MockDGAModel.java | 83 ++ .../maas/StellarMaaSIntegrationTest.java | 178 +++ .../enrichment/bolt/ThreatIntelJoinBolt.java | 23 +- .../triage/ThreatTriageProcessor.java | 15 +- .../threatintel/triage/ThreatTriageTest.java | 6 +- .../metron/parsers/bolt/ParserBolt.java | 9 +- .../parsers/filters/BroMessageFilter.java | 3 +- .../parsers/filters/GenericMessageFilter.java | 3 +- .../metron/parsers/filters/QueryFilter.java | 8 +- .../parsers/interfaces/MessageFilter.java | 4 +- .../apache/metron/filters/FiltersTest.java | 9 +- .../metron/parsers/bolt/ParserBoltTest.java | 13 +- .../WriterBoltIntegrationTest.java | 3 +- .../pcap/filter/query/QueryPcapFilter.java | 4 +- 66 files changed, 2717 insertions(+), 863 deletions(-) create mode 100644 metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java create mode 100644 metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Context.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java create mode 100644 metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/MockDGAModel.java create mode 100644 metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java new file mode 100644 index 0000000000..0ba1a18561 --- /dev/null +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.config; + +import java.util.HashMap; +import java.util.Map; + +public class Endpoint { + String url; + Map endpoints = new HashMap(){{ + put("apply", "apply"); + }}; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Map getEndpoints() { + return endpoints; + } + + public void setEndpoints(Map endpoints) { + this.endpoints = endpoints; + } + + + @Override + public String toString() { + return "Endpoint{" + + "url='" + url + '\'' + + ", endpoints=" + endpoints + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Endpoint endpoint = (Endpoint) o; + + if (getUrl() != null ? !getUrl().equals(endpoint.getUrl()) : endpoint.getUrl() != null) return false; + return getEndpoints() != null ? getEndpoints().equals(endpoint.getEndpoints()) : endpoint.getEndpoints() == null; + + } + + @Override + public int hashCode() { + int result = getUrl() != null ? getUrl().hashCode() : 0; + result = 31 * result + (getEndpoints() != null ? getEndpoints().hashCode() : 0); + return result; + } +} diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java index 1f3d60f1e0..7b756b0b02 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java @@ -17,10 +17,12 @@ */ package org.apache.metron.maas.config; +import com.google.common.base.Joiner; + import java.io.Serializable; public class ModelEndpoint implements Serializable { - private String url; + private Endpoint endpoint; private String name; private String version; private String containerId; @@ -29,26 +31,22 @@ public String getContainerId() { return containerId; } - @Override - public String toString() { - return "ModelEndpoint{" + - "url='" + url + '\'' + - ", name='" + name + '\'' + - ", version='" + version + '\'' + - ", containerId='" + containerId + '\'' + - '}'; + public Endpoint getEndpoint() { + return endpoint; } - public void setContainerId(String containerId) { - this.containerId = containerId; + public void setEndpoint(Endpoint endpoint) { + this.endpoint = endpoint; } - public String getUrl() { - return url; + @Override + public String toString() { + return name + ":" + version + " @ " + endpoint.getUrl() + + " serving:\n\t" + Joiner.on("\n\t").join(getEndpoint().getEndpoints().entrySet()); } - public void setUrl(String url) { - this.url = url; + public void setContainerId(String containerId) { + this.containerId = containerId; } public String getName() { @@ -74,17 +72,19 @@ public boolean equals(Object o) { ModelEndpoint that = (ModelEndpoint) o; - if (getUrl() != null ? !getUrl().equals(that.getUrl()) : that.getUrl() != null) return false; + if (getEndpoint() != null ? !getEndpoint().equals(that.getEndpoint()) : that.getEndpoint() != null) return false; if (getName() != null ? !getName().equals(that.getName()) : that.getName() != null) return false; - return getVersion() != null ? getVersion().equals(that.getVersion()) : that.getVersion() == null; + if (getVersion() != null ? !getVersion().equals(that.getVersion()) : that.getVersion() != null) return false; + return getContainerId() != null ? getContainerId().equals(that.getContainerId()) : that.getContainerId() == null; } @Override public int hashCode() { - int result = getUrl() != null ? getUrl().hashCode() : 0; + int result = getEndpoint() != null ? getEndpoint().hashCode() : 0; result = 31 * result + (getName() != null ? getName().hashCode() : 0); result = 31 * result + (getVersion() != null ? getVersion().hashCode() : 0); + result = 31 * result + (getContainerId() != null ? getContainerId().hashCode() : 0); return result; } } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java index dc2f6e000a..319b7edcd1 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java @@ -17,6 +17,9 @@ */ package org.apache.metron.maas.discovery; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.curator.framework.CuratorFramework; @@ -30,6 +33,11 @@ import java.io.Closeable; import java.util.*; import java.util.concurrent.ThreadLocalRandom; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -40,8 +48,16 @@ public class ServiceDiscoverer implements Closeable{ private ServiceDiscovery serviceDiscovery; private Map> state = new HashMap<>(); private Map> containerToEndpoint = new HashMap<>(); + private Map modelToCurrentVersion = new HashMap<>(); + private Cache blacklist; public ServiceDiscoverer(CuratorFramework client, String root) { + blacklist = CacheBuilder.newBuilder() + .concurrencyLevel(4) + .weakKeys() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + JsonInstanceSerializer serializer = new JsonInstanceSerializer<>(ModelEndpoint.class); serviceDiscovery = ServiceDiscoveryBuilder.builder(ModelEndpoint.class) .client(client) @@ -76,12 +92,19 @@ public ServiceDiscovery getServiceDiscovery() { private void updateState() { Map> state = new HashMap<>(); + Map modelToVersion = new HashMap<>(); Map> containerToEndpoint = new HashMap<>(); try { for(String name : serviceDiscovery.queryForNames()) { for(ServiceInstance endpoint: serviceDiscovery.queryForInstances(name)) { ModelEndpoint ep = endpoint.getPayload(); LOG.info("Found model endpoint " + ep); + String currentVersion = modelToVersion.getOrDefault(ep.getName(), ep.getVersion()); + modelToVersion.put( ep.getName() + , currentVersion.compareTo(ep.getVersion()) < 0 + ? ep.getVersion() + : currentVersion + ); containerToEndpoint.put(ep.getContainerId(), endpoint); Model model = new Model(ep.getName(), ep.getVersion()); List endpoints = state.get(model); @@ -93,6 +116,7 @@ private void updateState() { } } rwLock.writeLock().lock(); + this.modelToCurrentVersion = modelToVersion; this.state = state; this.containerToEndpoint = containerToEndpoint; rwLock.writeLock().unlock(); @@ -143,14 +167,59 @@ public List getEndpoints(Model model) { } } + public void blacklist(ModelEndpoint endpoint) { + blacklist(toUrl(endpoint.getEndpoint().getUrl())); + } + + public void blacklist(URL url) { + rwLock.writeLock().lock(); + blacklist.put(url, true); + rwLock.writeLock().unlock(); + } + + public ModelEndpoint getEndpoint(String modelName) { + String version = null; + rwLock.readLock().lock(); + version = modelToCurrentVersion.get(modelName); + rwLock.readLock().unlock(); + if(version == null) { + throw new IllegalStateException("Unable to find version for " + modelName); + } + return getEndpoint(modelName, version); + } + + private static URL toUrl(String url) { + try { + return new URL(url); + } catch (MalformedURLException e) { + throw new IllegalStateException("Endpoint does not refer to an actual URL"); + } + } + + public ModelEndpoint getEndpoint(String modelName, String modelVersion) { + return getEndpoint(new Model(modelName, modelVersion)); + } public ModelEndpoint getEndpoint(Model model) { rwLock.readLock().lock(); try { List endpoints = state.get(model); ModelEndpoint ret = null; if(endpoints != null) { - int i = ThreadLocalRandom.current().nextInt(endpoints.size()); - ret = endpoints.get(i); + for(int j = 0;j < 10;++j) { + int i = ThreadLocalRandom.current().nextInt(endpoints.size()); + ret = endpoints.get(i); + try { + if (blacklist.asMap().containsKey(toUrl(ret.getEndpoint().getUrl()))) { + continue; + } + else { + return ret; + } + } + catch(IllegalStateException ise) { + + } + } } return ret; } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/ConfigUtil.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/ConfigUtil.java index cf1ae453a8..6dc33aafeb 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/ConfigUtil.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/ConfigUtil.java @@ -27,24 +27,7 @@ public enum ConfigUtil { INSTANCE; private static ThreadLocal _mapper = new ThreadLocal() { - /** - * Returns the current thread's "initial value" for this - * thread-local variable. This method will be invoked the first - * time a thread accesses the variable with the {@link #get} - * method, unless the thread previously invoked the {@link #set} - * method, in which case the {@code initialValue} method will not - * be invoked for the thread. Normally, this method is invoked at - * most once per thread, but it may be invoked again in case of - * subsequent invocations of {@link #remove} followed by {@link #get}. - *

- *

This implementation simply returns {@code null}; if the - * programmer desires thread-local variables to have an initial - * value other than {@code null}, {@code ThreadLocal} must be - * subclassed, and this method overridden. Typically, an - * anonymous inner class will be used. - * - * @return the initial value for this thread-local - */ + @Override protected ObjectMapper initialValue() { return new ObjectMapper(); diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java new file mode 100644 index 0000000000..e3b6d8e729 --- /dev/null +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.util; + +import com.google.common.collect.Iterables; +import com.google.common.io.CharStreams; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public enum RESTUtil { + INSTANCE; + public static ThreadLocal CLIENT = new ThreadLocal() { + + @Override + protected HttpClient initialValue() { + //TODO: Figure out connection management + return new DefaultHttpClient(); + } + }; + + public String getRESTJSONResults(URL endpointUrl, Map getArgs) throws IOException, URISyntaxException { String encodedParams = encodeParams(getArgs); + HttpGet get = new HttpGet(appendToUrl(endpointUrl, encodedParams).toURI()); + get.addHeader("accept", "application/json"); + HttpResponse response = CLIENT.get().execute(get); + + if (response.getStatusLine().getStatusCode() != 200) { + throw new IllegalStateException("Failed : HTTP error code : " + + response.getStatusLine().getStatusCode()); + } + + return new BufferedReader(new InputStreamReader(response.getEntity().getContent())) + .lines().collect(Collectors.joining("\n")); + } + public URL appendToUrl(URL endpointUrl, String params) throws MalformedURLException { + return new URL(endpointUrl.toString() + "?" + params); + } + public String encodeParams(Map params) { + Iterable nvp = Iterables.transform(params.entrySet() + , kv -> new BasicNameValuePair(kv.getKey(), kv.getValue()) + ); + + return URLEncodedUtils.format(nvp, Charset.defaultCharset()); + } +} diff --git a/metron-analytics/metron-maas-common/src/test/java/org/apache/metron/maas/discovery/ServiceDiscoveryIntegrationTest.java b/metron-analytics/metron-maas-common/src/test/java/org/apache/metron/maas/discovery/ServiceDiscoveryIntegrationTest.java index 2453c6c06a..6eeb64a1ca 100644 --- a/metron-analytics/metron-maas-common/src/test/java/org/apache/metron/maas/discovery/ServiceDiscoveryIntegrationTest.java +++ b/metron-analytics/metron-maas-common/src/test/java/org/apache/metron/maas/discovery/ServiceDiscoveryIntegrationTest.java @@ -26,6 +26,7 @@ import org.apache.curator.x.discovery.ServiceInstance; import org.apache.curator.x.discovery.ServiceInstanceBuilder; import org.apache.curator.x.discovery.ServiceType; +import org.apache.metron.maas.config.Endpoint; import org.apache.metron.maas.config.Model; import org.apache.metron.maas.config.ModelEndpoint; import org.junit.After; @@ -54,7 +55,7 @@ public void setup() throws Exception { } private ServiceInstance createInstance(ModelEndpoint ep) throws Exception { - URL url = new URL(ep.getUrl()); + URL url = new URL(ep.getEndpoint().getUrl()); ServiceInstanceBuilder builder = ServiceInstance. builder() .address(url.getHost()) .id(ep.getContainerId()) @@ -75,7 +76,10 @@ private void registerService(String name, String version, AtomicInteger containe ep.setName(name); ep.setVersion(version); ep.setContainerId(containerId.incrementAndGet() + ""); - ep.setUrl("http://localhost:9080/ep1"); + ep.setEndpoint(new Endpoint() {{ + setUrl("http://localhost:9080/ep1"); + }} + ); registerService(ep); } @Test diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java index c5bb73b36e..33e460c622 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java @@ -32,6 +32,7 @@ import org.apache.curator.utils.CloseableUtils; import org.apache.curator.x.discovery.*; import org.apache.curator.x.discovery.details.JsonInstanceSerializer; +import org.apache.metron.maas.config.Endpoint; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.config.MaaSConfig; import org.apache.metron.maas.config.ModelEndpoint; @@ -214,13 +215,15 @@ public static void main(String... argv) throws Exception { try { LOG.info("Started " + cmd); - URL endpointUrl = correctLocalUrl(hostname, readURL(cwd)); - LOG.info("Read endpoint " + endpointUrl); + Endpoint ep = readEndpoint(cwd); + URL endpointUrl =correctLocalUrl(hostname, ep.getUrl()); + ep.setUrl(endpointUrl.toString()); + LOG.info("Read endpoint " + ep); ModelEndpoint endpoint = new ModelEndpoint(); { endpoint.setName(name); endpoint.setContainerId(containerId); - endpoint.setUrl(endpointUrl.toString()); + endpoint.setEndpoint(ep); endpoint.setVersion(version); }; ServiceInstanceBuilder builder = ServiceInstance. builder() @@ -272,7 +275,8 @@ public void run() add("127.0.0.1"); add("0.0.0.0"); }}; - private static URL correctLocalUrl(String hostname, URL tmp) { + private static URL correctLocalUrl(String hostname, String tmpUrl) throws MalformedURLException { + URL tmp = new URL(tmpUrl); if(hostname != null && hostname.length() > 0 && localAddresses.contains(tmp.getHost())) { URL endpointUrl = null; try { @@ -286,7 +290,7 @@ private static URL correctLocalUrl(String hostname, URL tmp) { return tmp; } - private static URL readURL(File cwd) throws IOException, InterruptedException { + private static Endpoint readEndpoint(File cwd) throws Exception { String content = ""; File f = new File(cwd, "endpoint.dat"); for(int i = 0;i < NUM_ATTEMPTS;i++) { @@ -298,10 +302,11 @@ private static URL readURL(File cwd) throws IOException, InterruptedException { } if(content != null && content.length() > 0) { try { - URL url = new URL(content); - return url; + Endpoint ep = ConfigUtil.INSTANCE.read(content.getBytes(), Endpoint.class); + return ep; } - catch(MalformedURLException mue) { + catch(Exception ex) { + LOG.error("Unable to parse " + content + ": " + ex.getMessage(), ex); } } } diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java index 46961cb502..050f520e0b 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java @@ -203,7 +203,7 @@ else if(mode.equalsIgnoreCase("LIST")) { String modelTitle = "Model " + kv.getKey().getName() + " @ " + kv.getKey().getVersion(); System.out.println(modelTitle); for(ModelEndpoint endpoint : kv.getValue()){ - System.out.println("\t" + endpoint.getContainerId() + " at " + endpoint.getUrl()); + System.out.println(endpoint); } } } diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java index 658cf4d458..c9284d08bc 100644 --- a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java +++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java @@ -270,7 +270,7 @@ public void run() { try { List endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { - String output = makeRESTcall(new URL(endpoints.get(0).getUrl() + "/echo/casey")); + String output = makeRESTcall(new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; @@ -329,7 +329,7 @@ private void cleanup() { new BufferedReader(new InputStreamReader(p.getInputStream())); while ((line = input.readLine()) != null) { if(line.contains("dummy_rest.sh")) { - String pid = Iterables.get(Splitter.on(" ").split(line.replaceAll("\\s+", " ")), 0); + String pid = Iterables.get(Splitter.on(" ").split(line.replaceAll("\\s+", " ").trim()), 0); System.out.println("Killing " + pid + " from " + line); Runtime.getRuntime().exec("kill -9 " + pid); } diff --git a/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh b/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh index 2b739a5d30..791c303a80 100644 --- a/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh +++ b/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh @@ -3,7 +3,7 @@ rm -f out mkfifo out trap "rm -f out" EXIT -echo "http://localhost:1500" > endpoint.dat +echo "{ \"url\" : \"http://localhost:1500\", \"endpoints\" : { \"apply\" : \"echo\" } }" > endpoint.dat while true do cat out | nc -l 0.0.0.0 1500 > >( # parse the netcat output, to build the answer redirected to the pipe "out". diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index d6254022bf..f48e7c22f8 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -37,6 +37,11 @@ + + org.apache.metron + metron-maas-common + ${parent.version} + commons-validator commons-validator diff --git a/metron-platform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4 b/metron-platform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4 index 03e576f651..1ce299e37b 100644 --- a/metron-platform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4 +++ b/metron-platform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4 @@ -65,15 +65,19 @@ LT : '<'; LTE : '<='; GT : '>'; GTE : '>='; -QUESTION : '?' | 'THEN' | 'then'; -COLON : ':' | 'ELSE' | 'else'; +QUESTION : '?' ; +COLON : ':' ; IF : 'IF' | 'if'; +THEN : 'THEN' | 'then'; +ELSE : 'ELSE' | 'else'; +NULL : 'null' | 'NULL'; MINUS : '-'; PLUS : '+'; DIV : '/'; MUL : '*'; - +LBRACE : '{'; +RBRACE : '}'; LBRACKET : '['; RBRACKET : ']'; LPAREN : '(' ; @@ -104,20 +108,25 @@ WS : [ \r\t\u000C\n]+ -> skip ; transformation : transformation_expr EOF; transformation_expr: - comparison_expr # ComparisonExpression - | LPAREN transformation_expr RPAREN #TransformationExpr - | transformation_entity #TransformationEntity - | transformation_expr AND transformation_expr #LogicalExpressionAnd - | transformation_expr OR transformation_expr #LogicalExpressionOr - | NOT LPAREN transformation_expr RPAREN #NotFunc - | transformation_expr QUESTION transformation_expr COLON transformation_expr #TernaryFunc - | IF transformation_expr QUESTION transformation_expr COLON transformation_expr #TernaryFunc + conditional_expr #ConditionalExpr + | LPAREN transformation_expr RPAREN #TransformationExpr + | arithmetic_expr # ArithExpression + | transformation_entity #TransformationEntity + | comparison_expr # ComparisonExpression ; +conditional_expr : comparison_expr QUESTION transformation_expr COLON transformation_expr #TernaryFuncWithoutIf + | IF comparison_expr THEN transformation_expr ELSE transformation_expr #TernaryFuncWithIf + ; + comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator | identifier_operand IN identifier_operand #InExpression | identifier_operand NIN identifier_operand #NInExpression + | comparison_expr AND comparison_expr #LogicalExpressionAnd + | comparison_expr OR comparison_expr #LogicalExpressionOr + | NOT LPAREN comparison_expr RPAREN #NotFunc | LPAREN comparison_expr RPAREN # ComparisonExpressionParens + | identifier_operand #operand ; comparison_operand : identifier_operand #IdentifierOperand ; @@ -129,7 +138,8 @@ arith_operator_addition : (PLUS | MINUS) # ArithOp_plus ; arith_operator_mul : (MUL | DIV) # ArithOp_mul ; -func_args : op_list +func_args : LPAREN op_list RPAREN + | LPAREN RPAREN ; op_list : identifier_operand | op_list COMMA identifier_operand @@ -137,6 +147,13 @@ op_list : identifier_operand list_entity : LBRACKET op_list RBRACKET | LBRACKET RBRACKET; +kv_list : identifier_operand COLON transformation_expr + | kv_list COMMA identifier_operand ':' transformation_expr + ; + +map_entity : LBRACE kv_list RBRACE + | LBRACE RBRACE; + arithmetic_expr: arithmetic_expr_mul #ArithExpr_solo | arithmetic_expr PLUS arithmetic_expr_mul #ArithExpr_plus | arithmetic_expr MINUS arithmetic_expr_mul #ArithExpr_minus @@ -146,7 +163,7 @@ arithmetic_expr_mul : arithmetic_operands #ArithExpr_mul_solo | arithmetic_expr_mul DIV arithmetic_expr_mul #ArithExpr_div ; -functions : IDENTIFIER LPAREN func_args RPAREN #TransformationFunc +functions : IDENTIFIER func_args #TransformationFunc ; arithmetic_operands : functions #NumericFunctions | DOUBLE_LITERAL #DoubleLiteral @@ -158,5 +175,7 @@ identifier_operand : (TRUE | FALSE) # LogicalConst | arithmetic_expr #ArithmeticOperands | STRING_LITERAL # StringLiteral | list_entity #List + | map_entity #MapConst + | NULL #NullConst | EXISTS LPAREN IDENTIFIER RPAREN #ExistsFunc ; diff --git a/metron-platform/metron-common/src/main/java/Stellar.tokens b/metron-platform/metron-common/src/main/java/Stellar.tokens index 46d8534849..c9e92bd3dd 100644 --- a/metron-platform/metron-common/src/main/java/Stellar.tokens +++ b/metron-platform/metron-common/src/main/java/Stellar.tokens @@ -13,23 +13,28 @@ GTE=12 QUESTION=13 COLON=14 IF=15 -MINUS=16 -PLUS=17 -DIV=18 -MUL=19 -LBRACKET=20 -RBRACKET=21 -LPAREN=22 -RPAREN=23 -IN=24 -NIN=25 -EXISTS=26 -INT_LITERAL=27 -DOUBLE_LITERAL=28 -IDENTIFIER=29 -STRING_LITERAL=30 -COMMENT=31 -WS=32 +THEN=16 +ELSE=17 +NULL=18 +MINUS=19 +PLUS=20 +DIV=21 +MUL=22 +LBRACE=23 +RBRACE=24 +LBRACKET=25 +RBRACKET=26 +LPAREN=27 +RPAREN=28 +IN=29 +NIN=30 +EXISTS=31 +INT_LITERAL=32 +DOUBLE_LITERAL=33 +IDENTIFIER=34 +STRING_LITERAL=35 +COMMENT=36 +WS=37 ','=1 '=='=7 '!='=8 @@ -37,13 +42,17 @@ WS=32 '<='=10 '>'=11 '>='=12 -'-'=16 -'+'=17 -'/'=18 -'*'=19 -'['=20 -']'=21 -'('=22 -')'=23 -'in'=24 -'not in'=25 +'?'=13 +':'=14 +'-'=19 +'+'=20 +'/'=21 +'*'=22 +'{'=23 +'}'=24 +'['=25 +']'=26 +'('=27 +')'=28 +'in'=29 +'not in'=30 diff --git a/metron-platform/metron-common/src/main/java/StellarLexer.tokens b/metron-platform/metron-common/src/main/java/StellarLexer.tokens index 46d8534849..c9e92bd3dd 100644 --- a/metron-platform/metron-common/src/main/java/StellarLexer.tokens +++ b/metron-platform/metron-common/src/main/java/StellarLexer.tokens @@ -13,23 +13,28 @@ GTE=12 QUESTION=13 COLON=14 IF=15 -MINUS=16 -PLUS=17 -DIV=18 -MUL=19 -LBRACKET=20 -RBRACKET=21 -LPAREN=22 -RPAREN=23 -IN=24 -NIN=25 -EXISTS=26 -INT_LITERAL=27 -DOUBLE_LITERAL=28 -IDENTIFIER=29 -STRING_LITERAL=30 -COMMENT=31 -WS=32 +THEN=16 +ELSE=17 +NULL=18 +MINUS=19 +PLUS=20 +DIV=21 +MUL=22 +LBRACE=23 +RBRACE=24 +LBRACKET=25 +RBRACKET=26 +LPAREN=27 +RPAREN=28 +IN=29 +NIN=30 +EXISTS=31 +INT_LITERAL=32 +DOUBLE_LITERAL=33 +IDENTIFIER=34 +STRING_LITERAL=35 +COMMENT=36 +WS=37 ','=1 '=='=7 '!='=8 @@ -37,13 +42,17 @@ WS=32 '<='=10 '>'=11 '>='=12 -'-'=16 -'+'=17 -'/'=18 -'*'=19 -'['=20 -']'=21 -'('=22 -')'=23 -'in'=24 -'not in'=25 +'?'=13 +':'=14 +'-'=19 +'+'=20 +'/'=21 +'*'=22 +'{'=23 +'}'=24 +'['=25 +']'=26 +'('=27 +')'=28 +'in'=29 +'not in'=30 diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java index 8c2ac143ca..fd51505fbe 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java @@ -31,6 +31,7 @@ import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.dsl.Context; import java.io.IOException; import java.util.Map; @@ -43,6 +44,7 @@ public abstract class ConfiguredBolt extends Ba protected CuratorFramework client; protected TreeCache cache; + protected org.apache.metron.common.dsl.Context context; private final CONFIG_T configurations = defaultConfigurations(); public ConfiguredBolt(String zookeeperUrl) { this.zookeeperUrl = zookeeperUrl; @@ -71,6 +73,9 @@ public void prepare(Map stormConf, TopologyContext context, OutputCollector coll client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); } client.start(); + this.context = new Context.Builder() + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) + .build(); if (cache == null) { cache = new TreeCache(client, Constants.ZOOKEEPER_TOPOLOGY_ROOT); TreeCacheListener listener = new TreeCacheListener() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index 4e34e049b5..c5ca7a2684 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.transformation.FieldTransformation; import org.apache.metron.common.field.transformation.FieldTransformations; import org.json.simple.JSONObject; @@ -105,21 +106,21 @@ public void initAndValidate() { } } - public Map transform(JSONObject input, Map sensorConfig) { + public Map transform(JSONObject input, Map sensorConfig, Context context) { if(getInput() == null || getInput().isEmpty()) { - return transformation.map(input, getOutput(), config, sensorConfig); + return transformation.map(input, getOutput(), config, sensorConfig, context); } else { Map in = new HashMap<>(); for(String inputField : getInput()) { in.put(inputField, input.get(inputField)); } - return transformation.map(in, getOutput(), config, sensorConfig); + return transformation.map(in, getOutput(), config, sensorConfig, context); } } - public void transformAndUpdate(JSONObject message, Map sensorConfig) { - Map currentValue = transform(message, sensorConfig); + public void transformAndUpdate(JSONObject message, Map sensorConfig, Context context) { + Map currentValue = transform(message, sensorConfig, context); if(currentValue != null) { for(Map.Entry kv : currentValue.entrySet()) { if(kv.getValue() == null) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java index 80d770a617..3302426ad0 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java @@ -19,6 +19,7 @@ package org.apache.metron.common.configuration; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import org.apache.metron.common.field.validation.FieldValidations; import org.json.simple.JSONObject; @@ -93,7 +94,7 @@ public Map getConfig() { return config; } - public boolean isValid(JSONObject inputData, Map globalConfig) { + public boolean isValid(JSONObject inputData, Map globalConfig, Context context) { Map in = inputData; if(input != null && !input.isEmpty()) { in = new HashMap<>(); @@ -102,7 +103,7 @@ public boolean isValid(JSONObject inputData, Map globalConfig) { in.put(i,o); } } - return validation.isValid(in, config, globalConfig); + return validation.isValid(in, config, globalConfig, context); } public static List readValidations(Map globalConfig) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java new file mode 100644 index 0000000000..0d8a207c7d --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl; + +import java.util.List; +import java.util.Map; + +public abstract class BaseStellarFunction implements StellarFunction { + public abstract Object apply(List args); + + @Override + public Object apply(List args, Context context) throws ParseException { + return apply(args); + } + + @Override + public void initialize(Context context) { + + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Context.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Context.java new file mode 100644 index 0000000000..8080363989 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Context.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +public class Context { + public interface Capability { + Object get(); + } + public enum Capabilities { + HBASE_PROVIDER + , ZOOKEEPER_CLIENT + , SERVICE_DISCOVERER; + + } + + public static class Builder { + private Map capabilityMap = new HashMap<>(); + + public Builder with(String s, Capability capability) { + capabilityMap.put(s, capability); + return this; + } + public Builder with(Enum s, Capability capability) { + capabilityMap.put(s.toString(), capability); + return this; + } + public Context build() { + + return new Context(capabilityMap); + } + } + public static Context EMPTY_CONTEXT() { + return + new Context(new HashMap<>()){ + @Override + public Optional getCapability(String capability) { + return Optional.empty(); + } + }; + } + private Map capabilities; + private Context( Map capabilities + ) + { + this.capabilities = capabilities; + } + public Optional getCapability(Enum capability) { + return getCapability(capability.toString()); + } + public Optional getCapability(String capability) { + Capability c = capabilities.get(capability); + if(c == null) { + throw new IllegalStateException("Unable to find capability " + capability + "; it may not be available in your context."); + } + return Optional.ofNullable(c.get()); + } + + public void addCapability(String s, Capability capability) { + this.capabilities.put(s, capability); + } + + public void addCapability(Enum s, Capability capability) { + this.capabilities.put(s.toString(), capability); + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java new file mode 100644 index 0000000000..917944999a --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl; + +import java.util.function.Function; + +public interface FunctionResolver extends Function { + void initializeFunctions(Context context); +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java new file mode 100644 index 0000000000..82b5fee262 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl; + +import java.util.List; + +public interface StellarFunction { + Object apply(List args, Context context) throws ParseException; + void initialize(Context context); +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java index 0394d36e54..d7057e84bb 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java @@ -19,10 +19,7 @@ package org.apache.metron.common.dsl; import org.apache.commons.net.util.SubnetUtils; -import org.apache.metron.common.dsl.functions.DateFunctions; -import org.apache.metron.common.dsl.functions.MapFunctions; -import org.apache.metron.common.dsl.functions.NetworkFunctions; -import org.apache.metron.common.dsl.functions.StringFunctions; +import org.apache.metron.common.dsl.functions.*; import org.apache.metron.common.field.transformation.IPProtocolTransformation; import org.apache.metron.common.field.validation.network.DomainValidation; import org.apache.metron.common.field.validation.network.EmailValidation; @@ -37,13 +34,13 @@ import java.util.function.Function; import java.util.function.Predicate; -public enum StellarFunctions implements Function, Object> { - TO_LOWER(strings -> strings.get(0)==null?null:strings.get(0).toString().toLowerCase()) - ,TO_UPPER(strings -> strings.get(0) == null?null:strings.get(0).toString().toUpperCase()) - ,TO_STRING(strings -> strings.get(0) == null?null:strings.get(0).toString()) - ,TO_INTEGER(strings -> strings.get(0) == null?null: ConversionUtils.convert(strings.get(0), Integer.class)) - ,TO_DOUBLE(strings -> strings.get(0) == null?null: ConversionUtils.convert(strings.get(0), Double.class)) - ,TRIM(strings -> strings.get(0) == null?null:strings.get(0).toString().trim()) +public enum StellarFunctions implements StellarFunction { + TO_LOWER(new StringFunctions.ToLower()) + ,TO_UPPER(new StringFunctions.ToUpper()) + ,TO_STRING(new StringFunctions.ToString()) + ,TO_INTEGER(new ConversionFunctions.Cast<>(Integer.class)) + ,TO_DOUBLE(new ConversionFunctions.Cast<>(Double.class)) + ,TRIM(new StringFunctions.Trim()) ,JOIN(new StringFunctions.JoinFunction()) ,SPLIT(new StringFunctions.SplitFunction()) ,GET_FIRST(new StringFunctions.GetFirst()) @@ -59,87 +56,22 @@ public enum StellarFunctions implements Function, Object> { ,URL_TO_PROTOCOL(new NetworkFunctions.URLToProtocol()) ,TO_EPOCH_TIMESTAMP(new DateFunctions.ToTimestamp()) ,PROTOCOL_TO_NAME(new IPProtocolTransformation()) - ,IS_EMPTY ( list -> { - if(list.size() == 0) { - throw new IllegalStateException("IS_EMPTY expects one string arg"); - } - String val = (String) list.get(0); - return val == null || val.isEmpty() ? true:false; - }) - ,IN_SUBNET( list -> { - if(list.size() < 2) { - throw new IllegalStateException("IN_SUBNET expects at least two args: [ip, cidr1, cidr2, ...]" - + " where cidr is the subnet mask in cidr form" - ); - } - String ip = (String) list.get(0); - if(ip == null) { - return false; - } - boolean inSubnet = false; - for(int i = 1;i < list.size() && !inSubnet;++i) { - String cidr = (String) list.get(1); - if(cidr == null) { - continue; - } - inSubnet |= new SubnetUtils(cidr).getInfo().isInRange(ip); - } - - return inSubnet; - }) - ,STARTS_WITH( list -> { - if(list.size() < 2) { - throw new IllegalStateException("STARTS_WITH expects two args: [string, prefix] where prefix is the string fragment that the string should start with"); - } - String prefix = (String) list.get(1); - String str = (String) list.get(0); - if(str == null || prefix == null) { - return false; - } - return str.startsWith(prefix); - }) - ,ENDS_WITH( list -> { - if(list.size() < 2) { - throw new IllegalStateException("ENDS_WITH expects two args: [string, suffix] where suffix is the string fragment that the string should end with"); - } - String prefix = (String) list.get(1); - String str = (String) list.get(0); - if(str == null || prefix == null) { - return false; - } - return str.endsWith(prefix); - }) - ,REGEXP_MATCH( list -> { - if(list.size() < 2) { - throw new IllegalStateException("REGEXP_MATCH expects two args: [string, pattern] where pattern is a regexp pattern"); - } - String pattern = (String) list.get(1); - String str = (String) list.get(0); - if(str == null || pattern == null) { - return false; - } - return str.matches(pattern); - }) + ,IS_EMPTY ( new DataStructureFunctions.IsEmpty()) + ,IN_SUBNET( new NetworkFunctions.InSubnet()) + ,STARTS_WITH( new StringFunctions.StartsWith()) + ,ENDS_WITH( new StringFunctions.EndsWith()) + ,REGEXP_MATCH( new StringFunctions.RegexpMatch()) , IS_IP(new Predicate2Transformation(new IPValidation())) , IS_DOMAIN(new Predicate2Transformation(new DomainValidation())) , IS_EMAIL(new Predicate2Transformation(new EmailValidation())) , IS_URL(new Predicate2Transformation(new URLValidation())) , IS_DATE(new Predicate2Transformation(new DateValidation())) , IS_INTEGER(new Predicate2Transformation(new IntegerValidation())) - , MAP_EXISTS(list -> { - if(list.size() < 2) { - return false; - } - Object key = list.get(0); - Object mapObj = list.get(1); - if(key != null && mapObj != null && mapObj instanceof Map) { - return ((Map)mapObj).containsKey(key); - } - return false; - } - ) + , MAP_EXISTS( new MapFunctions.MapExists()) + , MAAS_GET_ENDPOINT( new MaaSFunctions.GetEndpoint()) + , MODEL_APPLY(new MaaSFunctions.ModelApply()) ; - private static class Predicate2Transformation implements Function, Object> { + private static class Predicate2Transformation extends BaseStellarFunction { Predicate> pred; public Predicate2Transformation(Predicate> pred) { this.pred = pred; @@ -150,15 +82,43 @@ public Object apply(List objects) { return pred.test(objects); } } - Function, Object> func; - StellarFunctions(Function, Object> func) { + StellarFunction func; + StellarFunctions(StellarFunction func) { this.func = func; } @Override - public Object apply(List input) { - return func.apply(input); + public Object apply(List input, Context context) { + return func.apply(input, context); + } + @Override + public void initialize(Context context) { + func.initialize(context); + } + + public static FunctionResolver FUNCTION_RESOLVER() { + return new FunctionResolver() { + @Override + public void initializeFunctions(Context context) { + for(StellarFunctions s : StellarFunctions.values()) { + s.initialize(context); + } + } + + @Override + public StellarFunction apply(String s) { + StellarFunctions func = null; + try { + func = StellarFunctions.valueOf(s); + return func; + } + catch(Exception e) { + throw new IllegalStateException("Unable to resolve function " + s); + } + } + }; + } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java new file mode 100644 index 0000000000..0a712ce5ec --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl.functions; + +import org.apache.metron.common.dsl.BaseStellarFunction; +import org.apache.metron.common.utils.ConversionUtils; + +import java.util.List; + +public class ConversionFunctions { + public static class Cast extends BaseStellarFunction { + Class clazz; + public Cast(Class clazz) { + this.clazz = clazz; + } + + @Override + public Object apply(List strings ) { + return strings.get(0) == null?null: ConversionUtils.convert(strings.get(0), clazz); + } + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java new file mode 100644 index 0000000000..d7764ca01b --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl.functions; + +import org.apache.metron.common.dsl.BaseStellarFunction; + +import java.util.Collection; +import java.util.List; + +public class DataStructureFunctions { + public static class IsEmpty extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if(list.size() == 0) { + throw new IllegalStateException("IS_EMPTY expects one string arg"); + } + Object o = list.get(0); + if(o instanceof Collection) { + return ((Collection)o).isEmpty(); + } + else if(o instanceof String) { + String val = (String) list.get(0); + return val == null || val.isEmpty() ? true : false; + } + else { + throw new IllegalStateException("IS_EMPTY expects a collection or string"); + } + } + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java index f4a2b9724b..6008cfc48b 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java @@ -21,6 +21,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import org.apache.metron.common.dsl.BaseStellarFunction; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -102,7 +103,7 @@ public static long getEpochTime(String date, String format, Optional tim } - public static class ToTimestamp implements Function, Object> { + public static class ToTimestamp extends BaseStellarFunction { @Override public Object apply(List objects) { Object dateObj = objects.get(0); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java new file mode 100644 index 0000000000..9d14a26d6f --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -0,0 +1,259 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.dsl.functions; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.curator.framework.CuratorFramework; +import org.apache.hadoop.security.authorize.Service; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.MaaSConfig; +import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.ConfigUtil; +import org.apache.metron.maas.util.RESTUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +public class MaaSFunctions { + protected static final Logger LOG = LoggerFactory.getLogger(MaaSFunctions.class); + private static class ModelCacheKey { + String name; + String version; + String method; + Map args; + public ModelCacheKey(String name, String version, String method, Map args) { + this.name = name; + this.version = version; + this.method = method; + this.args = args; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ModelCacheKey that = (ModelCacheKey) o; + + if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (version != null ? !version.equals(that.version) : that.version != null) return false; + if (method != null ? !method.equals(that.method) : that.method != null) return false; + return args != null ? args.equals(that.args) : that.args == null; + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (method != null ? method.hashCode() : 0); + result = 31 * result + (args != null ? args.hashCode() : 0); + return result; + } + } + public static class ModelApply implements StellarFunction { + private ServiceDiscoverer discoverer; + private Cache > resultCache; + public ModelApply() { + resultCache = CacheBuilder.newBuilder() + .concurrencyLevel(4) + .weakKeys() + .maximumSize(100000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + } + + @Override + public Object apply(List args, Context context) throws ParseException { + if(args.size() < 2) { + throw new ParseException("Unable to execute model_apply. " + + "Expected arguments: endpoint_map:map, " + + " [endpoint method:string], model_args:map" + ); + } + int i = 0; + if(args.size() == 0) { + return null; + } + Object endpointObj = args.get(i++); + Map endpoint = null; + String modelName; + String modelVersion; + String modelUrl; + if(endpointObj instanceof Map) { + endpoint = (Map)endpointObj; + modelName = endpoint.get("name") + ""; + modelVersion = endpoint.get("version") + ""; + modelUrl = endpoint.get("url") + ""; + } + else { + return null; + } + String modelFunction = "apply"; + Map modelArgs = new HashMap<>(); + if(args.get(i) instanceof String) { + String func = (String)args.get(i); + if(endpoint.containsKey("endpoint:" + func)) { + modelFunction = "" + endpoint.get("endpoint:" + func); + } + else { + modelFunction = func; + } + i++; + } + + if(args.get(i) instanceof Map) { + if(endpoint.containsKey("endpoint:apply")) { + modelFunction = "" + endpoint.get("endpoint:apply"); + } + modelArgs = (Map)args.get(i); + } + if( modelName == null + || modelVersion == null + || modelFunction == null + ) { + return null; + } + ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs); + Map ret = resultCache.getIfPresent(cacheKey); + if(ret != null) { + return ret; + } + else { + String url = modelUrl; + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + if (modelFunction.startsWith("/")) { + modelFunction = modelFunction.substring(1); + } + try { + URL u = new URL(url + "/" + modelFunction); + + String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs); + ret = JSONUtils.INSTANCE.load(results, new TypeReference>() { + }); + resultCache.put(cacheKey, ret); + return ret; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + if (discoverer != null) { + try { + URL u = new URL(modelUrl); + discoverer.blacklist(u); + } catch (MalformedURLException e1) { + } + } + } + } + return null; + } + + @Override + public void initialize(Context context) { + try { + Optional discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); + if (discovererOpt.isPresent()) { + discoverer = discovererOpt.get(); + } + } + catch(Exception ex) { + LOG.error(ex.getMessage(), ex); + } + } + } + + public static class GetEndpoint implements StellarFunction { + ServiceDiscoverer discoverer; + @Override + public Object apply(List args, Context context) throws ParseException { + if(discoverer == null) { + throw new ParseException("Unable to find ServiceDiscoverer service..."); + } + String modelName = null; + String modelVersion = null; + if(args.size() >= 1) { + modelName = args.get(0).toString(); + } + if(args.size() >= 2) + { + modelVersion = args.get(1).toString(); + } + if(modelName == null) { + return null; + } + try { + ModelEndpoint ep = null; + if (modelVersion == null) { + ep = discoverer.getEndpoint(modelName); + } else { + ep = discoverer.getEndpoint(modelName, modelVersion); + } + return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); + } + catch(Exception ex) { + LOG.error("Unable to discover endpoint: " + ex.getMessage(), ex); + return null; + } + } + public static Map endpointToMap(String name, String version, Endpoint ep) { + Map ret = new HashMap<>(); + ret.put("url", ep.getUrl()); + ret.put("name", name); + ret.put("version", version); + for(Map.Entry kv : ep.getEndpoints().entrySet()) { + ret.put("endpoint:" + kv.getKey(), kv.getValue()); + } + return ret; + } + @Override + public void initialize(Context context) { + Optional clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if(clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework)clientOptional.get(); + } + else { + return; + } + try { + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return; + } + } + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java index 5d43416b82..0e322b7350 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java @@ -18,12 +18,29 @@ package org.apache.metron.common.dsl.functions; +import org.apache.metron.common.dsl.BaseStellarFunction; + import java.util.List; import java.util.Map; import java.util.function.Function; public class MapFunctions { - public static class MapGet implements Function, Object> { + public static class MapExists extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if(list.size() < 2) { + return false; + } + Object key = list.get(0); + Object mapObj = list.get(1); + if(key != null && mapObj != null && mapObj instanceof Map) { + return ((Map)mapObj).containsKey(key); + } + return false; + } + } + public static class MapGet extends BaseStellarFunction { @Override public Object apply(List objects) { Object keyObj = objects.get(0); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java index e8cda443ca..e6a1818146 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java @@ -21,6 +21,8 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.net.InternetDomainName; +import org.apache.commons.net.util.SubnetUtils; +import org.apache.metron.common.dsl.BaseStellarFunction; import java.net.MalformedURLException; import java.net.URL; @@ -28,7 +30,32 @@ import java.util.function.Function; public class NetworkFunctions { - public static class RemoveSubdomains implements Function, Object> { + public static class InSubnet extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if(list.size() < 2) { + throw new IllegalStateException("IN_SUBNET expects at least two args: [ip, cidr1, cidr2, ...]" + + " where cidr is the subnet mask in cidr form" + ); + } + String ip = (String) list.get(0); + if(ip == null) { + return false; + } + boolean inSubnet = false; + for(int i = 1;i < list.size() && !inSubnet;++i) { + String cidr = (String) list.get(1); + if(cidr == null) { + continue; + } + inSubnet |= new SubnetUtils(cidr).getInfo().isInRange(ip); + } + + return inSubnet; + } + } + public static class RemoveSubdomains extends BaseStellarFunction { @Override public Object apply(List objects) { @@ -54,7 +81,7 @@ public Object apply(List objects) { return null; } } - public static class RemoveTLD implements Function, Object> { + public static class RemoveTLD extends BaseStellarFunction { @Override public Object apply(List objects) { Object dnObj = objects.get(0); @@ -75,7 +102,7 @@ public Object apply(List objects) { } } - public static class ExtractTLD implements Function, Object> { + public static class ExtractTLD extends BaseStellarFunction { @Override public Object apply(List objects) { Object dnObj = objects.get(0); @@ -87,7 +114,7 @@ public Object apply(List objects) { } } - public static class URLToPort implements Function, Object> { + public static class URLToPort extends BaseStellarFunction { @Override public Object apply(List objects) { URL url = toUrl(objects.get(0)); @@ -99,14 +126,14 @@ public Object apply(List objects) { } } - public static class URLToPath implements Function, Object> { + public static class URLToPath extends BaseStellarFunction { @Override public Object apply(List objects) { URL url = toUrl(objects.get(0)); return url == null?null:url.getPath(); } } - public static class URLToHost implements Function, Object> { + public static class URLToHost extends BaseStellarFunction { @Override public Object apply(List objects) { @@ -115,7 +142,7 @@ public Object apply(List objects) { } } - public static class URLToProtocol implements Function, Object> { + public static class URLToProtocol extends BaseStellarFunction { @Override public Object apply(List objects) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java index 2793f02c83..beca43f6a3 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java @@ -21,13 +21,83 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import org.apache.metron.common.dsl.BaseStellarFunction; import java.util.ArrayList; import java.util.List; import java.util.function.Function; public class StringFunctions { - public static class JoinFunction implements Function, Object> { + public static class RegexpMatch extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if(list.size() < 2) { + throw new IllegalStateException("REGEXP_MATCH expects two args: [string, pattern] where pattern is a regexp pattern"); + } + String pattern = (String) list.get(1); + String str = (String) list.get(0); + if(str == null || pattern == null) { + return false; + } + return str.matches(pattern); + } + } + + public static class EndsWith extends BaseStellarFunction { + @Override + public Object apply(List list) { + if(list.size() < 2) { + throw new IllegalStateException("ENDS_WITH expects two args: [string, suffix] where suffix is the string fragment that the string should end with"); + } + String prefix = (String) list.get(1); + String str = (String) list.get(0); + if(str == null || prefix == null) { + return false; + } + return str.endsWith(prefix); + } + } + public static class StartsWith extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if(list.size() < 2) { + throw new IllegalStateException("STARTS_WITH expects two args: [string, prefix] where prefix is the string fragment that the string should start with"); + } + String prefix = (String) list.get(1); + String str = (String) list.get(0); + if(str == null || prefix == null) { + return false; + } + return str.startsWith(prefix); + } + } + public static class ToLower extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0)==null?null:strings.get(0).toString().toLowerCase(); + } + } + public static class ToUpper extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0)==null?null:strings.get(0).toString().toUpperCase(); + } + } + public static class ToString extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0)==null?null:strings.get(0).toString(); + } + } + public static class Trim extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0)==null?null:strings.get(0).toString().trim(); + } + } + public static class JoinFunction extends BaseStellarFunction { @Override public Object apply(List args) { List arg1 = (List) args.get(0); @@ -35,7 +105,7 @@ public Object apply(List args) { return Joiner.on(delim).join(Iterables.filter(arg1, x -> x != null)); } } - public static class SplitFunction implements Function, Object> { + public static class SplitFunction extends BaseStellarFunction { @Override public Object apply(List args) { List ret = new ArrayList(); @@ -49,14 +119,14 @@ public Object apply(List args) { } } - public static class GetLast implements Function, Object> { + public static class GetLast extends BaseStellarFunction { @Override public Object apply(List args) { List arg1 = (List) args.get(0); return Iterables.getLast(arg1, null); } } - public static class GetFirst implements Function, Object> { + public static class GetFirst extends BaseStellarFunction { @Override public Object apply(List args) { List arg1 = (List) args.get(0); @@ -64,7 +134,7 @@ public Object apply(List args) { } } - public static class Get implements Function, Object> { + public static class Get extends BaseStellarFunction { @Override public Object apply(List args) { List arg1 = (List) args.get(0); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java index e308dd3a20..a19d165ff8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java @@ -18,6 +18,8 @@ package org.apache.metron.common.field.transformation; +import org.apache.metron.common.dsl.Context; + import java.io.Serializable; import java.util.List; import java.util.Map; @@ -27,5 +29,6 @@ Map map( Map input , List outputField , Map fieldMappingConfig , Map sensorConfig + , Context context ); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java index c789018e31..0a4137a7ce 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java @@ -20,6 +20,9 @@ import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.StellarFunction; import org.apache.metron.common.utils.ConversionUtils; import java.util.HashMap; @@ -27,7 +30,7 @@ import java.util.Map; import java.util.function.Function; -public class IPProtocolTransformation extends SimpleFieldTransformation implements Function, Object> { +public class IPProtocolTransformation extends SimpleFieldTransformation implements StellarFunction { private final static Map PROTOCOLS = new HashMap<>(); @@ -183,7 +186,7 @@ public Map map(Object value, String outputField) { @Override - public Object apply(List objects) { + public Object apply(List objects, Context context) throws ParseException { Object keyObj = objects.get(0); if(keyObj == null) { return keyObj; @@ -198,4 +201,9 @@ public Object apply(List objects) { } return ret; } + + @Override + public void initialize(Context context) { + + } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java index f2f34c04f0..be8b409226 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java @@ -18,20 +18,19 @@ package org.apache.metron.common.field.transformation; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.*; import org.apache.metron.common.stellar.StellarPredicateProcessor; -import org.apache.metron.common.dsl.VariableResolver; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; public class RemoveTransformation implements FieldTransformation { public static final String CONDITION_CONF = "condition"; public static final StellarPredicateProcessor PASSTHROUGH_PROCESSOR = new StellarPredicateProcessor() { @Override - public Boolean parse(String rule, VariableResolver resolver) { + public Boolean parse(String rule, VariableResolver resolver, Function functionResolver, Context context) { return true; } @@ -69,10 +68,11 @@ public Map map( Map input , final List outputFields , Map fieldMappingConfig , Map sensorConfig + , Context context ) { String condition = getCondition(fieldMappingConfig); StellarPredicateProcessor processor = getPredicateProcessor(condition); - if(processor.parse(condition, new MapVariableResolver(input))) { + if(processor.parse(condition, new MapVariableResolver(input), StellarFunctions.FUNCTION_RESOLVER(), context)) { return new HashMap() {{ for(String outputField : outputFields) { put(outputField, null); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java index 3f6215e830..86e6f513a9 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java @@ -19,6 +19,7 @@ package org.apache.metron.common.field.transformation; import com.google.common.collect.Iterables; +import org.apache.metron.common.dsl.Context; import java.util.List; import java.util.Map; @@ -29,6 +30,7 @@ public Map map (Map input , List outputField , Map fieldMappingConfig , Map sensorConfig + , Context context ) { Object value = (input == null || input.values() == null && input.values().isEmpty()) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index a8ad052e77..fdc2fb5921 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -18,7 +18,9 @@ package org.apache.metron.common.field.transformation; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.MapVariableResolver; +import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.dsl.VariableResolver; import org.apache.metron.common.stellar.StellarProcessor; @@ -32,6 +34,7 @@ public Map map( Map input , List outputField , Map fieldMappingConfig , Map sensorConfig + , Context context ) { Map ret = new HashMap<>(); @@ -40,7 +43,7 @@ public Map map( Map input for(String oField : outputField) { Object transformObj = fieldMappingConfig.get(oField); if(transformObj != null) { - Object o = processor.parse(transformObj.toString(), resolver); + Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), context); if (o != null) { ret.put(oField, o); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java index c0d4c40fd6..59ba9bb2cd 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java @@ -18,6 +18,8 @@ package org.apache.metron.common.field.validation; +import org.apache.metron.common.dsl.Context; + import java.io.Serializable; import java.util.Map; @@ -25,6 +27,7 @@ public interface FieldValidation extends Serializable { boolean isValid( Map input , Map validationConfig , Map globalConfig + , Context context ); void initialize(Map validationConfig, Map globalConfig); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java index f9060af4f2..ee4314a89e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java @@ -18,7 +18,9 @@ package org.apache.metron.common.field.validation; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.MapVariableResolver; +import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.stellar.StellarPredicateProcessor; import java.util.Map; @@ -45,6 +47,7 @@ public T get(Map config, Class clazz) { public boolean isValid( Map input , Map validationConfig , Map globalConfig + , Context context ) { String condition = Config.CONDITION.get(validationConfig, String.class); if(condition == null) { @@ -52,7 +55,7 @@ public boolean isValid( Map input } else { StellarPredicateProcessor processor = new StellarPredicateProcessor(); - return processor.parse(condition, new MapVariableResolver(input, validationConfig, globalConfig)); + return processor.parse(condition, new MapVariableResolver(input, validationConfig, globalConfig), StellarFunctions.FUNCTION_RESOLVER(), context); } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java index 13b922fa81..33f78965f8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java @@ -18,6 +18,8 @@ package org.apache.metron.common.field.validation; +import org.apache.metron.common.dsl.Context; + import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -27,6 +29,7 @@ public abstract class SimpleValidation implements FieldValidation, Predicate input , Map validationConfig , Map globalConfig + , Context context ) { Predicate predicate = getPredicate(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java index 0aa907518b..94c32a5cb6 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java @@ -19,6 +19,7 @@ package org.apache.metron.common.field.validation.network; import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import java.util.List; @@ -103,6 +104,7 @@ public boolean test(List strings) { public boolean isValid( Map input , Map validationConfig , Map globalConfig + , Context context ) { IPType type = IPType.get(Config.TYPE.get(validationConfig, String.class)); for(Object o : input.values()) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java index 85121e278b..d0dc43e715 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java @@ -18,6 +18,7 @@ package org.apache.metron.common.field.validation.primitive; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import java.text.ParseException; @@ -80,6 +81,7 @@ public T get(Map config, Class clazz) { public boolean isValid( Map input , Map validationConfig , Map globalConfig + , Context context ) { String format = Config.FORMAT.get(validationConfig, String.class); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java index 248eaff37b..fc44f40d7e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java @@ -18,6 +18,7 @@ package org.apache.metron.common.field.validation.primitive; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import java.util.Map; @@ -44,6 +45,7 @@ public T get(Map config, Class clazz) { public boolean isValid( Map input , Map validationConfig , Map globalConfig + , Context context ) { String regex = Config.REGEX.get(validationConfig, String.class); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java index ffd4b960fe..0ce374becf 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java @@ -21,12 +21,12 @@ import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.TokenStream; -import org.apache.metron.common.dsl.ErrorListener; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.common.dsl.*; import org.apache.metron.common.stellar.generated.StellarLexer; import org.apache.metron.common.stellar.generated.StellarParser; +import java.util.function.Function; + import static org.apache.commons.lang3.StringUtils.isEmpty; public class BaseStellarProcessor { @@ -34,7 +34,12 @@ public class BaseStellarProcessor { public BaseStellarProcessor(Class clazz) { this.clazz = clazz; } - public T parse(String rule, VariableResolver resolver) { + public T parse( String rule + , VariableResolver variableResolver + , Function functionResolver + , Context context + ) + { if (rule == null || isEmpty(rule.trim())) { return null; } @@ -45,7 +50,7 @@ public T parse(String rule, VariableResolver resolver) { TokenStream tokens = new CommonTokenStream(lexer); StellarParser parser = new StellarParser(tokens); - StellarCompiler treeBuilder = new StellarCompiler(resolver); + StellarCompiler treeBuilder = new StellarCompiler(variableResolver, functionResolver, context); parser.addParseListener(treeBuilder); parser.removeErrorListeners(); parser.addErrorListener(new ErrorListener()); @@ -58,7 +63,7 @@ public boolean validate(String rule) throws ParseException { } public boolean validate(String rule, boolean throwException) throws ParseException { try { - parse(rule, x -> null); + parse(rule, x -> null, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); return true; } catch(Throwable t) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java index 6d92a07f65..8a91992b28 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java @@ -29,10 +29,18 @@ import java.util.function.Function; public class StellarCompiler extends StellarBaseListener { - private VariableResolver resolver = null; + private Context context = null; private Stack tokenStack = new Stack<>(); - public StellarCompiler(VariableResolver resolver) { - this.resolver = resolver; + private Function functionResolver; + private VariableResolver variableResolver; + public StellarCompiler( VariableResolver variableResolver + , Function functionResolver + , Context context + ) + { + this.variableResolver = variableResolver; + this.functionResolver = functionResolver; + this.context = context; } @Override @@ -72,6 +80,12 @@ private Double getDouble(Token token) { } } + + @Override + public void exitNullConst(StellarParser.NullConstContext ctx) { + tokenStack.push(new Token<>(null, Object.class)); + } + @Override public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) { Token right = popStack(); @@ -103,7 +117,6 @@ public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) { tokenStack.push(new Token<>(l / r, Double.class)); } - @Override public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { Token right = popStack(); @@ -113,8 +126,7 @@ public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { tokenStack.push(new Token<>(l * r, Double.class)); } - @Override - public void exitTernaryFunc(StellarParser.TernaryFuncContext ctx) { + private void handleConditional() { Token elseExpr = popStack(); Token thenExpr = popStack(); Token ifExpr = popStack(); @@ -127,6 +139,18 @@ public void exitTernaryFunc(StellarParser.TernaryFuncContext ctx) { } } + + @Override + public void exitTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx) { + handleConditional(); + } + + + @Override + public void exitTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx) { + handleConditional(); + } + @Override public void exitInExpression(StellarParser.InExpressionContext ctx) { Token left = popStack(); @@ -150,7 +174,7 @@ public void exitNotFunc(StellarParser.NotFuncContext ctx) { @Override public void exitVariable(StellarParser.VariableContext ctx) { - tokenStack.push(new Token<>(resolver.resolve(ctx.getText()), Object.class)); + tokenStack.push(new Token<>(variableResolver.resolve(ctx.getText()), Object.class)); } @Override @@ -158,7 +182,6 @@ public void exitStringLiteral(StellarParser.StringLiteralContext ctx) { tokenStack.push(new Token<>(ctx.getText().substring(1, ctx.getText().length() - 1), String.class)); } - @Override public void exitIntLiteral(StellarParser.IntLiteralContext ctx) { tokenStack.push(new Token<>(Integer.parseInt(ctx.getText()), Integer.class)); @@ -214,15 +237,16 @@ private boolean booleanOp(Token left, Token right, BooleanOp op, String op @Override public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) { String funcName = ctx.getChild(0).getText(); - Function, Object> func; + StellarFunction func = null; try { - func = StellarFunctions.valueOf(funcName); + func = functionResolver.apply(funcName); } - catch(IllegalArgumentException iae) { + catch(Exception iae) { throw new ParseException("Unable to find string function " + funcName + ". Valid functions are " + Joiner.on(',').join(StellarFunctions.values()) ); } + Token left = popStack(); List argList = null; if(left.getUnderlyingType().equals(List.class)) { @@ -231,7 +255,7 @@ public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) else { throw new ParseException("Unable to process in clause because " + left.getValue() + " is not a set"); } - Object result = func.apply(argList); + Object result = func.apply(argList, context); tokenStack.push(new Token<>(result, Object.class)); } @@ -239,7 +263,7 @@ public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) @Override public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) { String variable = ctx.getChild(2).getText(); - boolean exists = resolver.resolve(variable) != null; + boolean exists = variableResolver.resolve(variable) != null; tokenStack.push(new Token<>(exists, Boolean.class)); } @@ -248,7 +272,6 @@ public void enterFunc_args(StellarParser.Func_argsContext ctx) { tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class)); } - @Override public void exitFunc_args(StellarParser.Func_argsContext ctx) { LinkedList args = new LinkedList<>(); @@ -264,7 +287,31 @@ public void exitFunc_args(StellarParser.Func_argsContext ctx) { tokenStack.push(new Token<>(args, List.class)); } + @Override + public void enterMap_entity(StellarParser.Map_entityContext ctx) { + tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class)); + } + @Override + public void exitMap_entity(StellarParser.Map_entityContext ctx) { + HashMap args = new HashMap<>(); + Object value = null; + for(int i = 0;true;i++) { + Token token = popStack(); + if(token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } + else { + if(i % 2 == 0) { + value = token.getValue(); + } + else { + args.put(token.getValue() + "" , value); + } + } + } + tokenStack.push(new Token<>(args, Map.class)); + } @Override public void exitList_entity(StellarParser.List_entityContext ctx) { LinkedList args = new LinkedList<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java index e60988071b..093a8d8922 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java @@ -19,8 +19,12 @@ package org.apache.metron.common.stellar; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.StellarFunction; import org.apache.metron.common.dsl.VariableResolver; +import java.util.function.Function; + import static org.apache.commons.lang3.StringUtils.isEmpty; /** @@ -29,6 +33,7 @@ * tool, the output of the stellar statement is always a boolean. In java parlance, this is like a * java.util.function.Predicate */ + public class StellarPredicateProcessor extends BaseStellarProcessor { public StellarPredicateProcessor() { @@ -36,10 +41,15 @@ public StellarPredicateProcessor() { } @Override - public Boolean parse(String rule, VariableResolver resolver) { + public Boolean parse( String rule + , VariableResolver variableResolver + , Function functionResolver + , Context context + ) + { if(rule == null || isEmpty(rule.trim())) { return true; } - return super.parse(rule, resolver); + return super.parse(rule, variableResolver, functionResolver, context); } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarBaseListener.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarBaseListener.java index f4ea0f40f8..69a8457fff 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarBaseListener.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarBaseListener.java @@ -49,37 +49,37 @@ public class StellarBaseListener implements StellarListener { * *

The default implementation does nothing.

*/ - @Override public void enterComparisonExpression(StellarParser.ComparisonExpressionContext ctx) { } + @Override public void enterConditionalExpr(StellarParser.ConditionalExprContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitComparisonExpression(StellarParser.ComparisonExpressionContext ctx) { } + @Override public void exitConditionalExpr(StellarParser.ConditionalExprContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterTernaryFunc(StellarParser.TernaryFuncContext ctx) { } + @Override public void enterTransformationExpr(StellarParser.TransformationExprContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitTernaryFunc(StellarParser.TernaryFuncContext ctx) { } + @Override public void exitTransformationExpr(StellarParser.TransformationExprContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterNotFunc(StellarParser.NotFuncContext ctx) { } + @Override public void enterArithExpression(StellarParser.ArithExpressionContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitNotFunc(StellarParser.NotFuncContext ctx) { } + @Override public void exitArithExpression(StellarParser.ArithExpressionContext ctx) { } /** * {@inheritDoc} * @@ -97,61 +97,61 @@ public class StellarBaseListener implements StellarListener { * *

The default implementation does nothing.

*/ - @Override public void enterArithExpression(StellarParser.ArithExpressionContext ctx) { } + @Override public void enterComparisonExpression(StellarParser.ComparisonExpressionContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitArithExpression(StellarParser.ArithExpressionContext ctx) { } + @Override public void exitComparisonExpression(StellarParser.ComparisonExpressionContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterTransformationExpr(StellarParser.TransformationExprContext ctx) { } + @Override public void enterTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitTransformationExpr(StellarParser.TransformationExprContext ctx) { } + @Override public void exitTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { } + @Override public void enterTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { } + @Override public void exitTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { } + @Override public void enterNotFunc(StellarParser.NotFuncContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { } + @Override public void exitNotFunc(StellarParser.NotFuncContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void enterComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { } + @Override public void enterComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { } + @Override public void exitComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx) { } /** * {@inheritDoc} * @@ -164,6 +164,30 @@ public class StellarBaseListener implements StellarListener { *

The default implementation does nothing.

*/ @Override public void exitInExpression(StellarParser.InExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { } /** * {@inheritDoc} * @@ -181,13 +205,25 @@ public class StellarBaseListener implements StellarListener { * *

The default implementation does nothing.

*/ - @Override public void enterComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx) { } + @Override public void enterLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx) { } + @Override public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterOperand(StellarParser.OperandContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitOperand(StellarParser.OperandContext ctx) { } /** * {@inheritDoc} * @@ -284,6 +320,30 @@ public class StellarBaseListener implements StellarListener { *

The default implementation does nothing.

*/ @Override public void exitList_entity(StellarParser.List_entityContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterKv_list(StellarParser.Kv_listContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitKv_list(StellarParser.Kv_listContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterMap_entity(StellarParser.Map_entityContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitMap_entity(StellarParser.Map_entityContext ctx) { } /** * {@inheritDoc} * @@ -476,6 +536,30 @@ public class StellarBaseListener implements StellarListener { *

The default implementation does nothing.

*/ @Override public void exitList(StellarParser.ListContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterMapConst(StellarParser.MapConstContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitMapConst(StellarParser.MapConstContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterNullConst(StellarParser.NullConstContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitNullConst(StellarParser.NullConstContext ctx) { } /** * {@inheritDoc} * diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarLexer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarLexer.java index 21c17e3de4..1f655adad6 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarLexer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarLexer.java @@ -38,33 +38,33 @@ public class StellarLexer extends Lexer { new PredictionContextCache(); public static final int COMMA=1, AND=2, OR=3, NOT=4, TRUE=5, FALSE=6, EQ=7, NEQ=8, LT=9, LTE=10, - GT=11, GTE=12, QUESTION=13, COLON=14, IF=15, MINUS=16, PLUS=17, DIV=18, - MUL=19, LBRACKET=20, RBRACKET=21, LPAREN=22, RPAREN=23, IN=24, NIN=25, - EXISTS=26, INT_LITERAL=27, DOUBLE_LITERAL=28, IDENTIFIER=29, STRING_LITERAL=30, - COMMENT=31, WS=32; + GT=11, GTE=12, QUESTION=13, COLON=14, IF=15, THEN=16, ELSE=17, NULL=18, + MINUS=19, PLUS=20, DIV=21, MUL=22, LBRACE=23, RBRACE=24, LBRACKET=25, + RBRACKET=26, LPAREN=27, RPAREN=28, IN=29, NIN=30, EXISTS=31, INT_LITERAL=32, + DOUBLE_LITERAL=33, IDENTIFIER=34, STRING_LITERAL=35, COMMENT=36, WS=37; public static String[] modeNames = { "DEFAULT_MODE" }; public static final String[] ruleNames = { "COMMA", "AND", "OR", "NOT", "TRUE", "FALSE", "EQ", "NEQ", "LT", "LTE", - "GT", "GTE", "QUESTION", "COLON", "IF", "MINUS", "PLUS", "DIV", "MUL", - "LBRACKET", "RBRACKET", "LPAREN", "RPAREN", "IN", "NIN", "EXISTS", "INT_LITERAL", - "DOUBLE_LITERAL", "IDENTIFIER", "SCHAR", "STRING_LITERAL", "COMMENT", - "WS" + "GT", "GTE", "QUESTION", "COLON", "IF", "THEN", "ELSE", "NULL", "MINUS", + "PLUS", "DIV", "MUL", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", "LPAREN", + "RPAREN", "IN", "NIN", "EXISTS", "INT_LITERAL", "DOUBLE_LITERAL", "IDENTIFIER", + "SCHAR", "STRING_LITERAL", "COMMENT", "WS" }; private static final String[] _LITERAL_NAMES = { null, "','", null, null, null, null, null, "'=='", "'!='", "'<'", "'<='", - "'>'", "'>='", null, null, null, "'-'", "'+'", "'/'", "'*'", "'['", "']'", - "'('", "')'", "'in'", "'not in'" + "'>'", "'>='", "'?'", "':'", null, null, null, null, "'-'", "'+'", "'/'", + "'*'", "'{'", "'}'", "'['", "']'", "'('", "')'", "'in'", "'not in'" }; private static final String[] _SYMBOLIC_NAMES = { null, "COMMA", "AND", "OR", "NOT", "TRUE", "FALSE", "EQ", "NEQ", "LT", - "LTE", "GT", "GTE", "QUESTION", "COLON", "IF", "MINUS", "PLUS", "DIV", - "MUL", "LBRACKET", "RBRACKET", "LPAREN", "RPAREN", "IN", "NIN", "EXISTS", - "INT_LITERAL", "DOUBLE_LITERAL", "IDENTIFIER", "STRING_LITERAL", "COMMENT", - "WS" + "LTE", "GT", "GTE", "QUESTION", "COLON", "IF", "THEN", "ELSE", "NULL", + "MINUS", "PLUS", "DIV", "MUL", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", + "LPAREN", "RPAREN", "IN", "NIN", "EXISTS", "INT_LITERAL", "DOUBLE_LITERAL", + "IDENTIFIER", "STRING_LITERAL", "COMMENT", "WS" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -121,97 +121,105 @@ public StellarLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2\"\u0110\b\1\4\2\t"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2\'\u012a\b\1\4\2\t"+ "\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13"+ "\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+ "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"+ - "\t!\4\"\t\"\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3P\n\3\3\4\3\4\3"+ - "\4\3\4\3\4\3\4\5\4X\n\4\3\5\3\5\3\5\3\5\3\5\3\5\5\5`\n\5\3\6\3\6\3\6\3"+ - "\6\3\6\3\6\3\6\3\6\5\6j\n\6\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\5"+ - "\7v\n\7\3\b\3\b\3\b\3\t\3\t\3\t\3\n\3\n\3\13\3\13\3\13\3\f\3\f\3\r\3\r"+ - "\3\r\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\5\16\u0091\n\16\3\17"+ - "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17\u009c\n\17\3\20\3\20\3\20"+ - "\3\20\5\20\u00a2\n\20\3\21\3\21\3\22\3\22\3\23\3\23\3\24\3\24\3\25\3\25"+ - "\3\26\3\26\3\27\3\27\3\30\3\30\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32"+ - "\3\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33"+ - "\5\33\u00ca\n\33\3\34\5\34\u00cd\n\34\3\34\6\34\u00d0\n\34\r\34\16\34"+ - "\u00d1\3\35\5\35\u00d5\n\35\3\35\6\35\u00d8\n\35\r\35\16\35\u00d9\3\35"+ - "\3\35\6\35\u00de\n\35\r\35\16\35\u00df\3\36\3\36\7\36\u00e4\n\36\f\36"+ - "\16\36\u00e7\13\36\3\37\3\37\3 \3 \7 \u00ed\n \f \16 \u00f0\13 \3 \3 "+ - "\3 \7 \u00f5\n \f \16 \u00f8\13 \3 \5 \u00fb\n \3!\3!\3!\3!\6!\u0101\n"+ - "!\r!\16!\u0102\3!\5!\u0106\n!\3!\3!\3\"\6\"\u010b\n\"\r\"\16\"\u010c\3"+ - "\"\3\"\3\u0102\2#\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31"+ - "\16\33\17\35\20\37\21!\22#\23%\24\'\25)\26+\27-\30/\31\61\32\63\33\65"+ - "\34\67\359\36;\37=\2? A!C\"\3\2\7\5\2C\\aac|\b\2\60\60\62;C\\^^aac|\7"+ - "\2\f\f\17\17$$))^^\3\3\f\f\5\2\13\f\16\17\"\"\u0126\2\3\3\2\2\2\2\5\3"+ - "\2\2\2\2\7\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3\2\2\2\2"+ - "\21\3\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3\2\2\2\2\33\3"+ - "\2\2\2\2\35\3\2\2\2\2\37\3\2\2\2\2!\3\2\2\2\2#\3\2\2\2\2%\3\2\2\2\2\'"+ - "\3\2\2\2\2)\3\2\2\2\2+\3\2\2\2\2-\3\2\2\2\2/\3\2\2\2\2\61\3\2\2\2\2\63"+ - "\3\2\2\2\2\65\3\2\2\2\2\67\3\2\2\2\29\3\2\2\2\2;\3\2\2\2\2?\3\2\2\2\2"+ - "A\3\2\2\2\2C\3\2\2\2\3E\3\2\2\2\5O\3\2\2\2\7W\3\2\2\2\t_\3\2\2\2\13i\3"+ - "\2\2\2\ru\3\2\2\2\17w\3\2\2\2\21z\3\2\2\2\23}\3\2\2\2\25\177\3\2\2\2\27"+ - "\u0082\3\2\2\2\31\u0084\3\2\2\2\33\u0090\3\2\2\2\35\u009b\3\2\2\2\37\u00a1"+ - "\3\2\2\2!\u00a3\3\2\2\2#\u00a5\3\2\2\2%\u00a7\3\2\2\2\'\u00a9\3\2\2\2"+ - ")\u00ab\3\2\2\2+\u00ad\3\2\2\2-\u00af\3\2\2\2/\u00b1\3\2\2\2\61\u00b3"+ - "\3\2\2\2\63\u00b6\3\2\2\2\65\u00c9\3\2\2\2\67\u00cc\3\2\2\29\u00d4\3\2"+ - "\2\2;\u00e1\3\2\2\2=\u00e8\3\2\2\2?\u00fa\3\2\2\2A\u00fc\3\2\2\2C\u010a"+ - "\3\2\2\2EF\7.\2\2F\4\3\2\2\2GH\7c\2\2HI\7p\2\2IP\7f\2\2JK\7(\2\2KP\7("+ - "\2\2LM\7C\2\2MN\7P\2\2NP\7F\2\2OG\3\2\2\2OJ\3\2\2\2OL\3\2\2\2P\6\3\2\2"+ - "\2QR\7q\2\2RX\7t\2\2ST\7~\2\2TX\7~\2\2UV\7Q\2\2VX\7T\2\2WQ\3\2\2\2WS\3"+ - "\2\2\2WU\3\2\2\2X\b\3\2\2\2YZ\7p\2\2Z[\7q\2\2[`\7v\2\2\\]\7P\2\2]^\7Q"+ - "\2\2^`\7V\2\2_Y\3\2\2\2_\\\3\2\2\2`\n\3\2\2\2ab\7v\2\2bc\7t\2\2cd\7w\2"+ - "\2dj\7g\2\2ef\7V\2\2fg\7T\2\2gh\7W\2\2hj\7G\2\2ia\3\2\2\2ie\3\2\2\2j\f"+ - "\3\2\2\2kl\7h\2\2lm\7c\2\2mn\7n\2\2no\7u\2\2ov\7g\2\2pq\7H\2\2qr\7C\2"+ - "\2rs\7N\2\2st\7U\2\2tv\7G\2\2uk\3\2\2\2up\3\2\2\2v\16\3\2\2\2wx\7?\2\2"+ - "xy\7?\2\2y\20\3\2\2\2z{\7#\2\2{|\7?\2\2|\22\3\2\2\2}~\7>\2\2~\24\3\2\2"+ - "\2\177\u0080\7>\2\2\u0080\u0081\7?\2\2\u0081\26\3\2\2\2\u0082\u0083\7"+ - "@\2\2\u0083\30\3\2\2\2\u0084\u0085\7@\2\2\u0085\u0086\7?\2\2\u0086\32"+ - "\3\2\2\2\u0087\u0091\7A\2\2\u0088\u0089\7V\2\2\u0089\u008a\7J\2\2\u008a"+ - "\u008b\7G\2\2\u008b\u0091\7P\2\2\u008c\u008d\7v\2\2\u008d\u008e\7j\2\2"+ - "\u008e\u008f\7g\2\2\u008f\u0091\7p\2\2\u0090\u0087\3\2\2\2\u0090\u0088"+ - "\3\2\2\2\u0090\u008c\3\2\2\2\u0091\34\3\2\2\2\u0092\u009c\7<\2\2\u0093"+ - "\u0094\7G\2\2\u0094\u0095\7N\2\2\u0095\u0096\7U\2\2\u0096\u009c\7G\2\2"+ - "\u0097\u0098\7g\2\2\u0098\u0099\7n\2\2\u0099\u009a\7u\2\2\u009a\u009c"+ - "\7g\2\2\u009b\u0092\3\2\2\2\u009b\u0093\3\2\2\2\u009b\u0097\3\2\2\2\u009c"+ - "\36\3\2\2\2\u009d\u009e\7K\2\2\u009e\u00a2\7H\2\2\u009f\u00a0\7k\2\2\u00a0"+ - "\u00a2\7h\2\2\u00a1\u009d\3\2\2\2\u00a1\u009f\3\2\2\2\u00a2 \3\2\2\2\u00a3"+ - "\u00a4\7/\2\2\u00a4\"\3\2\2\2\u00a5\u00a6\7-\2\2\u00a6$\3\2\2\2\u00a7"+ - "\u00a8\7\61\2\2\u00a8&\3\2\2\2\u00a9\u00aa\7,\2\2\u00aa(\3\2\2\2\u00ab"+ - "\u00ac\7]\2\2\u00ac*\3\2\2\2\u00ad\u00ae\7_\2\2\u00ae,\3\2\2\2\u00af\u00b0"+ - "\7*\2\2\u00b0.\3\2\2\2\u00b1\u00b2\7+\2\2\u00b2\60\3\2\2\2\u00b3\u00b4"+ - "\7k\2\2\u00b4\u00b5\7p\2\2\u00b5\62\3\2\2\2\u00b6\u00b7\7p\2\2\u00b7\u00b8"+ - "\7q\2\2\u00b8\u00b9\7v\2\2\u00b9\u00ba\7\"\2\2\u00ba\u00bb\7k\2\2\u00bb"+ - "\u00bc\7p\2\2\u00bc\64\3\2\2\2\u00bd\u00be\7g\2\2\u00be\u00bf\7z\2\2\u00bf"+ - "\u00c0\7k\2\2\u00c0\u00c1\7u\2\2\u00c1\u00c2\7v\2\2\u00c2\u00ca\7u\2\2"+ - "\u00c3\u00c4\7G\2\2\u00c4\u00c5\7Z\2\2\u00c5\u00c6\7K\2\2\u00c6\u00c7"+ - "\7U\2\2\u00c7\u00c8\7V\2\2\u00c8\u00ca\7U\2\2\u00c9\u00bd\3\2\2\2\u00c9"+ - "\u00c3\3\2\2\2\u00ca\66\3\2\2\2\u00cb\u00cd\5!\21\2\u00cc\u00cb\3\2\2"+ - "\2\u00cc\u00cd\3\2\2\2\u00cd\u00cf\3\2\2\2\u00ce\u00d0\4\62;\2\u00cf\u00ce"+ - "\3\2\2\2\u00d0\u00d1\3\2\2\2\u00d1\u00cf\3\2\2\2\u00d1\u00d2\3\2\2\2\u00d2"+ - "8\3\2\2\2\u00d3\u00d5\5!\21\2\u00d4\u00d3\3\2\2\2\u00d4\u00d5\3\2\2\2"+ - "\u00d5\u00d7\3\2\2\2\u00d6\u00d8\4\62;\2\u00d7\u00d6\3\2\2\2\u00d8\u00d9"+ - "\3\2\2\2\u00d9\u00d7\3\2\2\2\u00d9\u00da\3\2\2\2\u00da\u00db\3\2\2\2\u00db"+ - "\u00dd\7\60\2\2\u00dc\u00de\4\62;\2\u00dd\u00dc\3\2\2\2\u00de\u00df\3"+ - "\2\2\2\u00df\u00dd\3\2\2\2\u00df\u00e0\3\2\2\2\u00e0:\3\2\2\2\u00e1\u00e5"+ - "\t\2\2\2\u00e2\u00e4\t\3\2\2\u00e3\u00e2\3\2\2\2\u00e4\u00e7\3\2\2\2\u00e5"+ - "\u00e3\3\2\2\2\u00e5\u00e6\3\2\2\2\u00e6<\3\2\2\2\u00e7\u00e5\3\2\2\2"+ - "\u00e8\u00e9\n\4\2\2\u00e9>\3\2\2\2\u00ea\u00ee\7$\2\2\u00eb\u00ed\5="+ - "\37\2\u00ec\u00eb\3\2\2\2\u00ed\u00f0\3\2\2\2\u00ee\u00ec\3\2\2\2\u00ee"+ - "\u00ef\3\2\2\2\u00ef\u00f1\3\2\2\2\u00f0\u00ee\3\2\2\2\u00f1\u00fb\7$"+ - "\2\2\u00f2\u00f6\7)\2\2\u00f3\u00f5\5=\37\2\u00f4\u00f3\3\2\2\2\u00f5"+ - "\u00f8\3\2\2\2\u00f6\u00f4\3\2\2\2\u00f6\u00f7\3\2\2\2\u00f7\u00f9\3\2"+ - "\2\2\u00f8\u00f6\3\2\2\2\u00f9\u00fb\7)\2\2\u00fa\u00ea\3\2\2\2\u00fa"+ - "\u00f2\3\2\2\2\u00fb@\3\2\2\2\u00fc\u00fd\7\61\2\2\u00fd\u00fe\7\61\2"+ - "\2\u00fe\u0100\3\2\2\2\u00ff\u0101\13\2\2\2\u0100\u00ff\3\2\2\2\u0101"+ - "\u0102\3\2\2\2\u0102\u0103\3\2\2\2\u0102\u0100\3\2\2\2\u0103\u0105\3\2"+ - "\2\2\u0104\u0106\t\5\2\2\u0105\u0104\3\2\2\2\u0106\u0107\3\2\2\2\u0107"+ - "\u0108\b!\2\2\u0108B\3\2\2\2\u0109\u010b\t\6\2\2\u010a\u0109\3\2\2\2\u010b"+ - "\u010c\3\2\2\2\u010c\u010a\3\2\2\2\u010c\u010d\3\2\2\2\u010d\u010e\3\2"+ - "\2\2\u010e\u010f\b\"\2\2\u010fD\3\2\2\2\30\2OW_iu\u0090\u009b\u00a1\u00c9"+ - "\u00cc\u00d1\u00d4\u00d9\u00df\u00e5\u00ee\u00f6\u00fa\u0102\u0105\u010c"+ - "\3\b\2\2"; + "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\3\2\3\2\3\3\3\3\3\3\3\3\3"+ + "\3\3\3\3\3\3\3\5\3Z\n\3\3\4\3\4\3\4\3\4\3\4\3\4\5\4b\n\4\3\5\3\5\3\5\3"+ + "\5\3\5\3\5\5\5j\n\5\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\5\6t\n\6\3\7\3\7\3"+ + "\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\5\7\u0080\n\7\3\b\3\b\3\b\3\t\3\t\3\t\3"+ + "\n\3\n\3\13\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\17\3\17\3\20\3\20"+ + "\3\20\3\20\5\20\u009a\n\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21"+ + "\u00a4\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u00ae\n\22\3"+ + "\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u00b8\n\23\3\24\3\24\3\25"+ + "\3\25\3\26\3\26\3\27\3\27\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34"+ + "\3\34\3\35\3\35\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3 \3"+ + " \3 \3 \3 \3 \3 \3 \3 \3 \3 \3 \5 \u00e4\n \3!\5!\u00e7\n!\3!\6!\u00ea"+ + "\n!\r!\16!\u00eb\3\"\5\"\u00ef\n\"\3\"\6\"\u00f2\n\"\r\"\16\"\u00f3\3"+ + "\"\3\"\6\"\u00f8\n\"\r\"\16\"\u00f9\3#\3#\7#\u00fe\n#\f#\16#\u0101\13"+ + "#\3$\3$\3%\3%\7%\u0107\n%\f%\16%\u010a\13%\3%\3%\3%\7%\u010f\n%\f%\16"+ + "%\u0112\13%\3%\5%\u0115\n%\3&\3&\3&\3&\6&\u011b\n&\r&\16&\u011c\3&\5&"+ + "\u0120\n&\3&\3&\3\'\6\'\u0125\n\'\r\'\16\'\u0126\3\'\3\'\3\u011c\2(\3"+ + "\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31\16\33\17\35\20\37"+ + "\21!\22#\23%\24\'\25)\26+\27-\30/\31\61\32\63\33\65\34\67\359\36;\37="+ + " ?!A\"C#E$G\2I%K&M\'\3\2\7\5\2C\\aac|\b\2\60\60\62;C\\^^aac|\7\2\f\f\17"+ + "\17$$))^^\3\3\f\f\5\2\13\f\16\17\"\"\u013f\2\3\3\2\2\2\2\5\3\2\2\2\2\7"+ + "\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3\2\2\2\2\21\3\2\2"+ + "\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3\2\2\2\2\33\3\2\2\2\2"+ + "\35\3\2\2\2\2\37\3\2\2\2\2!\3\2\2\2\2#\3\2\2\2\2%\3\2\2\2\2\'\3\2\2\2"+ + "\2)\3\2\2\2\2+\3\2\2\2\2-\3\2\2\2\2/\3\2\2\2\2\61\3\2\2\2\2\63\3\2\2\2"+ + "\2\65\3\2\2\2\2\67\3\2\2\2\29\3\2\2\2\2;\3\2\2\2\2=\3\2\2\2\2?\3\2\2\2"+ + "\2A\3\2\2\2\2C\3\2\2\2\2E\3\2\2\2\2I\3\2\2\2\2K\3\2\2\2\2M\3\2\2\2\3O"+ + "\3\2\2\2\5Y\3\2\2\2\7a\3\2\2\2\ti\3\2\2\2\13s\3\2\2\2\r\177\3\2\2\2\17"+ + "\u0081\3\2\2\2\21\u0084\3\2\2\2\23\u0087\3\2\2\2\25\u0089\3\2\2\2\27\u008c"+ + "\3\2\2\2\31\u008e\3\2\2\2\33\u0091\3\2\2\2\35\u0093\3\2\2\2\37\u0099\3"+ + "\2\2\2!\u00a3\3\2\2\2#\u00ad\3\2\2\2%\u00b7\3\2\2\2\'\u00b9\3\2\2\2)\u00bb"+ + "\3\2\2\2+\u00bd\3\2\2\2-\u00bf\3\2\2\2/\u00c1\3\2\2\2\61\u00c3\3\2\2\2"+ + "\63\u00c5\3\2\2\2\65\u00c7\3\2\2\2\67\u00c9\3\2\2\29\u00cb\3\2\2\2;\u00cd"+ + "\3\2\2\2=\u00d0\3\2\2\2?\u00e3\3\2\2\2A\u00e6\3\2\2\2C\u00ee\3\2\2\2E"+ + "\u00fb\3\2\2\2G\u0102\3\2\2\2I\u0114\3\2\2\2K\u0116\3\2\2\2M\u0124\3\2"+ + "\2\2OP\7.\2\2P\4\3\2\2\2QR\7c\2\2RS\7p\2\2SZ\7f\2\2TU\7(\2\2UZ\7(\2\2"+ + "VW\7C\2\2WX\7P\2\2XZ\7F\2\2YQ\3\2\2\2YT\3\2\2\2YV\3\2\2\2Z\6\3\2\2\2["+ + "\\\7q\2\2\\b\7t\2\2]^\7~\2\2^b\7~\2\2_`\7Q\2\2`b\7T\2\2a[\3\2\2\2a]\3"+ + "\2\2\2a_\3\2\2\2b\b\3\2\2\2cd\7p\2\2de\7q\2\2ej\7v\2\2fg\7P\2\2gh\7Q\2"+ + "\2hj\7V\2\2ic\3\2\2\2if\3\2\2\2j\n\3\2\2\2kl\7v\2\2lm\7t\2\2mn\7w\2\2"+ + "nt\7g\2\2op\7V\2\2pq\7T\2\2qr\7W\2\2rt\7G\2\2sk\3\2\2\2so\3\2\2\2t\f\3"+ + "\2\2\2uv\7h\2\2vw\7c\2\2wx\7n\2\2xy\7u\2\2y\u0080\7g\2\2z{\7H\2\2{|\7"+ + "C\2\2|}\7N\2\2}~\7U\2\2~\u0080\7G\2\2\177u\3\2\2\2\177z\3\2\2\2\u0080"+ + "\16\3\2\2\2\u0081\u0082\7?\2\2\u0082\u0083\7?\2\2\u0083\20\3\2\2\2\u0084"+ + "\u0085\7#\2\2\u0085\u0086\7?\2\2\u0086\22\3\2\2\2\u0087\u0088\7>\2\2\u0088"+ + "\24\3\2\2\2\u0089\u008a\7>\2\2\u008a\u008b\7?\2\2\u008b\26\3\2\2\2\u008c"+ + "\u008d\7@\2\2\u008d\30\3\2\2\2\u008e\u008f\7@\2\2\u008f\u0090\7?\2\2\u0090"+ + "\32\3\2\2\2\u0091\u0092\7A\2\2\u0092\34\3\2\2\2\u0093\u0094\7<\2\2\u0094"+ + "\36\3\2\2\2\u0095\u0096\7K\2\2\u0096\u009a\7H\2\2\u0097\u0098\7k\2\2\u0098"+ + "\u009a\7h\2\2\u0099\u0095\3\2\2\2\u0099\u0097\3\2\2\2\u009a \3\2\2\2\u009b"+ + "\u009c\7V\2\2\u009c\u009d\7J\2\2\u009d\u009e\7G\2\2\u009e\u00a4\7P\2\2"+ + "\u009f\u00a0\7v\2\2\u00a0\u00a1\7j\2\2\u00a1\u00a2\7g\2\2\u00a2\u00a4"+ + "\7p\2\2\u00a3\u009b\3\2\2\2\u00a3\u009f\3\2\2\2\u00a4\"\3\2\2\2\u00a5"+ + "\u00a6\7G\2\2\u00a6\u00a7\7N\2\2\u00a7\u00a8\7U\2\2\u00a8\u00ae\7G\2\2"+ + "\u00a9\u00aa\7g\2\2\u00aa\u00ab\7n\2\2\u00ab\u00ac\7u\2\2\u00ac\u00ae"+ + "\7g\2\2\u00ad\u00a5\3\2\2\2\u00ad\u00a9\3\2\2\2\u00ae$\3\2\2\2\u00af\u00b0"+ + "\7p\2\2\u00b0\u00b1\7w\2\2\u00b1\u00b2\7n\2\2\u00b2\u00b8\7n\2\2\u00b3"+ + "\u00b4\7P\2\2\u00b4\u00b5\7W\2\2\u00b5\u00b6\7N\2\2\u00b6\u00b8\7N\2\2"+ + "\u00b7\u00af\3\2\2\2\u00b7\u00b3\3\2\2\2\u00b8&\3\2\2\2\u00b9\u00ba\7"+ + "/\2\2\u00ba(\3\2\2\2\u00bb\u00bc\7-\2\2\u00bc*\3\2\2\2\u00bd\u00be\7\61"+ + "\2\2\u00be,\3\2\2\2\u00bf\u00c0\7,\2\2\u00c0.\3\2\2\2\u00c1\u00c2\7}\2"+ + "\2\u00c2\60\3\2\2\2\u00c3\u00c4\7\177\2\2\u00c4\62\3\2\2\2\u00c5\u00c6"+ + "\7]\2\2\u00c6\64\3\2\2\2\u00c7\u00c8\7_\2\2\u00c8\66\3\2\2\2\u00c9\u00ca"+ + "\7*\2\2\u00ca8\3\2\2\2\u00cb\u00cc\7+\2\2\u00cc:\3\2\2\2\u00cd\u00ce\7"+ + "k\2\2\u00ce\u00cf\7p\2\2\u00cf<\3\2\2\2\u00d0\u00d1\7p\2\2\u00d1\u00d2"+ + "\7q\2\2\u00d2\u00d3\7v\2\2\u00d3\u00d4\7\"\2\2\u00d4\u00d5\7k\2\2\u00d5"+ + "\u00d6\7p\2\2\u00d6>\3\2\2\2\u00d7\u00d8\7g\2\2\u00d8\u00d9\7z\2\2\u00d9"+ + "\u00da\7k\2\2\u00da\u00db\7u\2\2\u00db\u00dc\7v\2\2\u00dc\u00e4\7u\2\2"+ + "\u00dd\u00de\7G\2\2\u00de\u00df\7Z\2\2\u00df\u00e0\7K\2\2\u00e0\u00e1"+ + "\7U\2\2\u00e1\u00e2\7V\2\2\u00e2\u00e4\7U\2\2\u00e3\u00d7\3\2\2\2\u00e3"+ + "\u00dd\3\2\2\2\u00e4@\3\2\2\2\u00e5\u00e7\5\'\24\2\u00e6\u00e5\3\2\2\2"+ + "\u00e6\u00e7\3\2\2\2\u00e7\u00e9\3\2\2\2\u00e8\u00ea\4\62;\2\u00e9\u00e8"+ + "\3\2\2\2\u00ea\u00eb\3\2\2\2\u00eb\u00e9\3\2\2\2\u00eb\u00ec\3\2\2\2\u00ec"+ + "B\3\2\2\2\u00ed\u00ef\5\'\24\2\u00ee\u00ed\3\2\2\2\u00ee\u00ef\3\2\2\2"+ + "\u00ef\u00f1\3\2\2\2\u00f0\u00f2\4\62;\2\u00f1\u00f0\3\2\2\2\u00f2\u00f3"+ + "\3\2\2\2\u00f3\u00f1\3\2\2\2\u00f3\u00f4\3\2\2\2\u00f4\u00f5\3\2\2\2\u00f5"+ + "\u00f7\7\60\2\2\u00f6\u00f8\4\62;\2\u00f7\u00f6\3\2\2\2\u00f8\u00f9\3"+ + "\2\2\2\u00f9\u00f7\3\2\2\2\u00f9\u00fa\3\2\2\2\u00faD\3\2\2\2\u00fb\u00ff"+ + "\t\2\2\2\u00fc\u00fe\t\3\2\2\u00fd\u00fc\3\2\2\2\u00fe\u0101\3\2\2\2\u00ff"+ + "\u00fd\3\2\2\2\u00ff\u0100\3\2\2\2\u0100F\3\2\2\2\u0101\u00ff\3\2\2\2"+ + "\u0102\u0103\n\4\2\2\u0103H\3\2\2\2\u0104\u0108\7$\2\2\u0105\u0107\5G"+ + "$\2\u0106\u0105\3\2\2\2\u0107\u010a\3\2\2\2\u0108\u0106\3\2\2\2\u0108"+ + "\u0109\3\2\2\2\u0109\u010b\3\2\2\2\u010a\u0108\3\2\2\2\u010b\u0115\7$"+ + "\2\2\u010c\u0110\7)\2\2\u010d\u010f\5G$\2\u010e\u010d\3\2\2\2\u010f\u0112"+ + "\3\2\2\2\u0110\u010e\3\2\2\2\u0110\u0111\3\2\2\2\u0111\u0113\3\2\2\2\u0112"+ + "\u0110\3\2\2\2\u0113\u0115\7)\2\2\u0114\u0104\3\2\2\2\u0114\u010c\3\2"+ + "\2\2\u0115J\3\2\2\2\u0116\u0117\7\61\2\2\u0117\u0118\7\61\2\2\u0118\u011a"+ + "\3\2\2\2\u0119\u011b\13\2\2\2\u011a\u0119\3\2\2\2\u011b\u011c\3\2\2\2"+ + "\u011c\u011d\3\2\2\2\u011c\u011a\3\2\2\2\u011d\u011f\3\2\2\2\u011e\u0120"+ + "\t\5\2\2\u011f\u011e\3\2\2\2\u0120\u0121\3\2\2\2\u0121\u0122\b&\2\2\u0122"+ + "L\3\2\2\2\u0123\u0125\t\6\2\2\u0124\u0123\3\2\2\2\u0125\u0126\3\2\2\2"+ + "\u0126\u0124\3\2\2\2\u0126\u0127\3\2\2\2\u0127\u0128\3\2\2\2\u0128\u0129"+ + "\b\'\2\2\u0129N\3\2\2\2\31\2Yais\177\u0099\u00a3\u00ad\u00b7\u00e3\u00e6"+ + "\u00eb\u00ee\u00f3\u00f9\u00ff\u0108\u0110\u0114\u011c\u011f\u0126\3\b"+ + "\2\2"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarListener.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarListener.java index 887233ad10..a41aad0407 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarListener.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarListener.java @@ -39,41 +39,41 @@ public interface StellarListener extends ParseTreeListener { */ void exitTransformation(StellarParser.TransformationContext ctx); /** - * Enter a parse tree produced by the {@code ComparisonExpression} + * Enter a parse tree produced by the {@code ConditionalExpr} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void enterComparisonExpression(StellarParser.ComparisonExpressionContext ctx); + void enterConditionalExpr(StellarParser.ConditionalExprContext ctx); /** - * Exit a parse tree produced by the {@code ComparisonExpression} + * Exit a parse tree produced by the {@code ConditionalExpr} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void exitComparisonExpression(StellarParser.ComparisonExpressionContext ctx); + void exitConditionalExpr(StellarParser.ConditionalExprContext ctx); /** - * Enter a parse tree produced by the {@code TernaryFunc} + * Enter a parse tree produced by the {@code TransformationExpr} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void enterTernaryFunc(StellarParser.TernaryFuncContext ctx); + void enterTransformationExpr(StellarParser.TransformationExprContext ctx); /** - * Exit a parse tree produced by the {@code TernaryFunc} + * Exit a parse tree produced by the {@code TransformationExpr} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void exitTernaryFunc(StellarParser.TernaryFuncContext ctx); + void exitTransformationExpr(StellarParser.TransformationExprContext ctx); /** - * Enter a parse tree produced by the {@code NotFunc} + * Enter a parse tree produced by the {@code ArithExpression} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void enterNotFunc(StellarParser.NotFuncContext ctx); + void enterArithExpression(StellarParser.ArithExpressionContext ctx); /** - * Exit a parse tree produced by the {@code NotFunc} + * Exit a parse tree produced by the {@code ArithExpression} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void exitNotFunc(StellarParser.NotFuncContext ctx); + void exitArithExpression(StellarParser.ArithExpressionContext ctx); /** * Enter a parse tree produced by the {@code TransformationEntity} * labeled alternative in {@link StellarParser#transformation_expr}. @@ -87,65 +87,65 @@ public interface StellarListener extends ParseTreeListener { */ void exitTransformationEntity(StellarParser.TransformationEntityContext ctx); /** - * Enter a parse tree produced by the {@code ArithExpression} + * Enter a parse tree produced by the {@code ComparisonExpression} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void enterArithExpression(StellarParser.ArithExpressionContext ctx); + void enterComparisonExpression(StellarParser.ComparisonExpressionContext ctx); /** - * Exit a parse tree produced by the {@code ArithExpression} + * Exit a parse tree produced by the {@code ComparisonExpression} * labeled alternative in {@link StellarParser#transformation_expr}. * @param ctx the parse tree */ - void exitArithExpression(StellarParser.ArithExpressionContext ctx); + void exitComparisonExpression(StellarParser.ComparisonExpressionContext ctx); /** - * Enter a parse tree produced by the {@code TransformationExpr} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Enter a parse tree produced by the {@code TernaryFuncWithoutIf} + * labeled alternative in {@link StellarParser#conditional_expr}. * @param ctx the parse tree */ - void enterTransformationExpr(StellarParser.TransformationExprContext ctx); + void enterTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx); /** - * Exit a parse tree produced by the {@code TransformationExpr} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Exit a parse tree produced by the {@code TernaryFuncWithoutIf} + * labeled alternative in {@link StellarParser#conditional_expr}. * @param ctx the parse tree */ - void exitTransformationExpr(StellarParser.TransformationExprContext ctx); + void exitTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx); /** - * Enter a parse tree produced by the {@code LogicalExpressionAnd} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Enter a parse tree produced by the {@code TernaryFuncWithIf} + * labeled alternative in {@link StellarParser#conditional_expr}. * @param ctx the parse tree */ - void enterLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx); + void enterTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx); /** - * Exit a parse tree produced by the {@code LogicalExpressionAnd} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Exit a parse tree produced by the {@code TernaryFuncWithIf} + * labeled alternative in {@link StellarParser#conditional_expr}. * @param ctx the parse tree */ - void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx); + void exitTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx); /** - * Enter a parse tree produced by the {@code LogicalExpressionOr} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Enter a parse tree produced by the {@code NotFunc} + * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void enterLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx); + void enterNotFunc(StellarParser.NotFuncContext ctx); /** - * Exit a parse tree produced by the {@code LogicalExpressionOr} - * labeled alternative in {@link StellarParser#transformation_expr}. + * Exit a parse tree produced by the {@code NotFunc} + * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx); + void exitNotFunc(StellarParser.NotFuncContext ctx); /** - * Enter a parse tree produced by the {@code ComparisonExpressionWithOperator} + * Enter a parse tree produced by the {@code ComparisonExpressionParens} * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void enterComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx); + void enterComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx); /** - * Exit a parse tree produced by the {@code ComparisonExpressionWithOperator} + * Exit a parse tree produced by the {@code ComparisonExpressionParens} * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx); + void exitComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx); /** * Enter a parse tree produced by the {@code InExpression} * labeled alternative in {@link StellarParser#comparison_expr}. @@ -158,6 +158,30 @@ public interface StellarListener extends ParseTreeListener { * @param ctx the parse tree */ void exitInExpression(StellarParser.InExpressionContext ctx); + /** + * Enter a parse tree produced by the {@code ComparisonExpressionWithOperator} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void enterComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx); + /** + * Exit a parse tree produced by the {@code ComparisonExpressionWithOperator} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx); + /** + * Enter a parse tree produced by the {@code LogicalExpressionAnd} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void enterLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx); + /** + * Exit a parse tree produced by the {@code LogicalExpressionAnd} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx); /** * Enter a parse tree produced by the {@code NInExpression} * labeled alternative in {@link StellarParser#comparison_expr}. @@ -171,17 +195,29 @@ public interface StellarListener extends ParseTreeListener { */ void exitNInExpression(StellarParser.NInExpressionContext ctx); /** - * Enter a parse tree produced by the {@code ComparisonExpressionParens} + * Enter a parse tree produced by the {@code LogicalExpressionOr} * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void enterComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx); + void enterLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx); /** - * Exit a parse tree produced by the {@code ComparisonExpressionParens} + * Exit a parse tree produced by the {@code LogicalExpressionOr} * labeled alternative in {@link StellarParser#comparison_expr}. * @param ctx the parse tree */ - void exitComparisonExpressionParens(StellarParser.ComparisonExpressionParensContext ctx); + void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx); + /** + * Enter a parse tree produced by the {@code operand} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void enterOperand(StellarParser.OperandContext ctx); + /** + * Exit a parse tree produced by the {@code operand} + * labeled alternative in {@link StellarParser#comparison_expr}. + * @param ctx the parse tree + */ + void exitOperand(StellarParser.OperandContext ctx); /** * Enter a parse tree produced by the {@code IdentifierOperand} * labeled alternative in {@link StellarParser#comparison_operand}. @@ -270,6 +306,26 @@ public interface StellarListener extends ParseTreeListener { * @param ctx the parse tree */ void exitList_entity(StellarParser.List_entityContext ctx); + /** + * Enter a parse tree produced by {@link StellarParser#kv_list}. + * @param ctx the parse tree + */ + void enterKv_list(StellarParser.Kv_listContext ctx); + /** + * Exit a parse tree produced by {@link StellarParser#kv_list}. + * @param ctx the parse tree + */ + void exitKv_list(StellarParser.Kv_listContext ctx); + /** + * Enter a parse tree produced by {@link StellarParser#map_entity}. + * @param ctx the parse tree + */ + void enterMap_entity(StellarParser.Map_entityContext ctx); + /** + * Exit a parse tree produced by {@link StellarParser#map_entity}. + * @param ctx the parse tree + */ + void exitMap_entity(StellarParser.Map_entityContext ctx); /** * Enter a parse tree produced by the {@code ArithExpr_solo} * labeled alternative in {@link StellarParser#arithmetic_expr}. @@ -462,6 +518,30 @@ public interface StellarListener extends ParseTreeListener { * @param ctx the parse tree */ void exitList(StellarParser.ListContext ctx); + /** + * Enter a parse tree produced by the {@code MapConst} + * labeled alternative in {@link StellarParser#identifier_operand}. + * @param ctx the parse tree + */ + void enterMapConst(StellarParser.MapConstContext ctx); + /** + * Exit a parse tree produced by the {@code MapConst} + * labeled alternative in {@link StellarParser#identifier_operand}. + * @param ctx the parse tree + */ + void exitMapConst(StellarParser.MapConstContext ctx); + /** + * Enter a parse tree produced by the {@code NullConst} + * labeled alternative in {@link StellarParser#identifier_operand}. + * @param ctx the parse tree + */ + void enterNullConst(StellarParser.NullConstContext ctx); + /** + * Exit a parse tree produced by the {@code NullConst} + * labeled alternative in {@link StellarParser#identifier_operand}. + * @param ctx the parse tree + */ + void exitNullConst(StellarParser.NullConstContext ctx); /** * Enter a parse tree produced by the {@code ExistsFunc} * labeled alternative in {@link StellarParser#identifier_operand}. diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarParser.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarParser.java index d3d603cff2..33eff6abce 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarParser.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarParser.java @@ -38,34 +38,36 @@ public class StellarParser extends Parser { new PredictionContextCache(); public static final int COMMA=1, AND=2, OR=3, NOT=4, TRUE=5, FALSE=6, EQ=7, NEQ=8, LT=9, LTE=10, - GT=11, GTE=12, QUESTION=13, COLON=14, IF=15, MINUS=16, PLUS=17, DIV=18, - MUL=19, LBRACKET=20, RBRACKET=21, LPAREN=22, RPAREN=23, IN=24, NIN=25, - EXISTS=26, INT_LITERAL=27, DOUBLE_LITERAL=28, IDENTIFIER=29, STRING_LITERAL=30, - COMMENT=31, WS=32; + GT=11, GTE=12, QUESTION=13, COLON=14, IF=15, THEN=16, ELSE=17, NULL=18, + MINUS=19, PLUS=20, DIV=21, MUL=22, LBRACE=23, RBRACE=24, LBRACKET=25, + RBRACKET=26, LPAREN=27, RPAREN=28, IN=29, NIN=30, EXISTS=31, INT_LITERAL=32, + DOUBLE_LITERAL=33, IDENTIFIER=34, STRING_LITERAL=35, COMMENT=36, WS=37; public static final int - RULE_transformation = 0, RULE_transformation_expr = 1, RULE_comparison_expr = 2, - RULE_comparison_operand = 3, RULE_transformation_entity = 4, RULE_comp_operator = 5, - RULE_arith_operator_addition = 6, RULE_arith_operator_mul = 7, RULE_func_args = 8, - RULE_op_list = 9, RULE_list_entity = 10, RULE_arithmetic_expr = 11, RULE_arithmetic_expr_mul = 12, - RULE_functions = 13, RULE_arithmetic_operands = 14, RULE_identifier_operand = 15; + RULE_transformation = 0, RULE_transformation_expr = 1, RULE_conditional_expr = 2, + RULE_comparison_expr = 3, RULE_comparison_operand = 4, RULE_transformation_entity = 5, + RULE_comp_operator = 6, RULE_arith_operator_addition = 7, RULE_arith_operator_mul = 8, + RULE_func_args = 9, RULE_op_list = 10, RULE_list_entity = 11, RULE_kv_list = 12, + RULE_map_entity = 13, RULE_arithmetic_expr = 14, RULE_arithmetic_expr_mul = 15, + RULE_functions = 16, RULE_arithmetic_operands = 17, RULE_identifier_operand = 18; public static final String[] ruleNames = { - "transformation", "transformation_expr", "comparison_expr", "comparison_operand", - "transformation_entity", "comp_operator", "arith_operator_addition", "arith_operator_mul", - "func_args", "op_list", "list_entity", "arithmetic_expr", "arithmetic_expr_mul", - "functions", "arithmetic_operands", "identifier_operand" + "transformation", "transformation_expr", "conditional_expr", "comparison_expr", + "comparison_operand", "transformation_entity", "comp_operator", "arith_operator_addition", + "arith_operator_mul", "func_args", "op_list", "list_entity", "kv_list", + "map_entity", "arithmetic_expr", "arithmetic_expr_mul", "functions", "arithmetic_operands", + "identifier_operand" }; private static final String[] _LITERAL_NAMES = { null, "','", null, null, null, null, null, "'=='", "'!='", "'<'", "'<='", - "'>'", "'>='", null, null, null, "'-'", "'+'", "'/'", "'*'", "'['", "']'", - "'('", "')'", "'in'", "'not in'" + "'>'", "'>='", "'?'", "':'", null, null, null, null, "'-'", "'+'", "'/'", + "'*'", "'{'", "'}'", "'['", "']'", "'('", "')'", "'in'", "'not in'" }; private static final String[] _SYMBOLIC_NAMES = { null, "COMMA", "AND", "OR", "NOT", "TRUE", "FALSE", "EQ", "NEQ", "LT", - "LTE", "GT", "GTE", "QUESTION", "COLON", "IF", "MINUS", "PLUS", "DIV", - "MUL", "LBRACKET", "RBRACKET", "LPAREN", "RPAREN", "IN", "NIN", "EXISTS", - "INT_LITERAL", "DOUBLE_LITERAL", "IDENTIFIER", "STRING_LITERAL", "COMMENT", - "WS" + "LTE", "GT", "GTE", "QUESTION", "COLON", "IF", "THEN", "ELSE", "NULL", + "MINUS", "PLUS", "DIV", "MUL", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", + "LPAREN", "RPAREN", "IN", "NIN", "EXISTS", "INT_LITERAL", "DOUBLE_LITERAL", + "IDENTIFIER", "STRING_LITERAL", "COMMENT", "WS" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -141,9 +143,9 @@ public final TransformationContext transformation() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(32); - transformation_expr(0); - setState(33); + setState(38); + transformation_expr(); + setState(39); match(EOF); } } @@ -183,43 +185,6 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitComparisonExpression(this); } } - public static class TernaryFuncContext extends Transformation_exprContext { - public TerminalNode IF() { return getToken(StellarParser.IF, 0); } - public List transformation_expr() { - return getRuleContexts(Transformation_exprContext.class); - } - public Transformation_exprContext transformation_expr(int i) { - return getRuleContext(Transformation_exprContext.class,i); - } - public TerminalNode QUESTION() { return getToken(StellarParser.QUESTION, 0); } - public TerminalNode COLON() { return getToken(StellarParser.COLON, 0); } - public TernaryFuncContext(Transformation_exprContext ctx) { copyFrom(ctx); } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).enterTernaryFunc(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).exitTernaryFunc(this); - } - } - public static class NotFuncContext extends Transformation_exprContext { - public TerminalNode NOT() { return getToken(StellarParser.NOT, 0); } - public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } - public Transformation_exprContext transformation_expr() { - return getRuleContext(Transformation_exprContext.class,0); - } - public TerminalNode RPAREN() { return getToken(StellarParser.RPAREN, 0); } - public NotFuncContext(Transformation_exprContext ctx) { copyFrom(ctx); } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).enterNotFunc(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).exitNotFunc(this); - } - } public static class TransformationEntityContext extends Transformation_exprContext { public Transformation_entityContext transformation_entity() { return getRuleContext(Transformation_entityContext.class,0); @@ -264,194 +229,194 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitTransformationExpr(this); } } - public static class LogicalExpressionAndContext extends Transformation_exprContext { - public List transformation_expr() { - return getRuleContexts(Transformation_exprContext.class); - } - public Transformation_exprContext transformation_expr(int i) { - return getRuleContext(Transformation_exprContext.class,i); - } - public TerminalNode AND() { return getToken(StellarParser.AND, 0); } - public LogicalExpressionAndContext(Transformation_exprContext ctx) { copyFrom(ctx); } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).enterLogicalExpressionAnd(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).exitLogicalExpressionAnd(this); - } - } - public static class LogicalExpressionOrContext extends Transformation_exprContext { - public List transformation_expr() { - return getRuleContexts(Transformation_exprContext.class); + public static class ConditionalExprContext extends Transformation_exprContext { + public Conditional_exprContext conditional_expr() { + return getRuleContext(Conditional_exprContext.class,0); } - public Transformation_exprContext transformation_expr(int i) { - return getRuleContext(Transformation_exprContext.class,i); - } - public TerminalNode OR() { return getToken(StellarParser.OR, 0); } - public LogicalExpressionOrContext(Transformation_exprContext ctx) { copyFrom(ctx); } + public ConditionalExprContext(Transformation_exprContext ctx) { copyFrom(ctx); } @Override public void enterRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).enterLogicalExpressionOr(this); + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterConditionalExpr(this); } @Override public void exitRule(ParseTreeListener listener) { - if ( listener instanceof StellarListener ) ((StellarListener)listener).exitLogicalExpressionOr(this); + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitConditionalExpr(this); } } public final Transformation_exprContext transformation_expr() throws RecognitionException { - return transformation_expr(0); - } - - private Transformation_exprContext transformation_expr(int _p) throws RecognitionException { - ParserRuleContext _parentctx = _ctx; - int _parentState = getState(); - Transformation_exprContext _localctx = new Transformation_exprContext(_ctx, _parentState); - Transformation_exprContext _prevctx = _localctx; - int _startState = 2; - enterRecursionRule(_localctx, 2, RULE_transformation_expr, _p); + Transformation_exprContext _localctx = new Transformation_exprContext(_ctx, getState()); + enterRule(_localctx, 2, RULE_transformation_expr); try { - int _alt; - enterOuterAlt(_localctx, 1); - { - setState(55); + setState(49); switch ( getInterpreter().adaptivePredict(_input,0,_ctx) ) { case 1: + _localctx = new ConditionalExprContext(_localctx); + enterOuterAlt(_localctx, 1); { - _localctx = new ComparisonExpressionContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; - - setState(36); - comparison_expr(); + setState(41); + conditional_expr(); } break; case 2: - { _localctx = new TransformationExprContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; - setState(37); + enterOuterAlt(_localctx, 2); + { + setState(42); match(LPAREN); - setState(38); - transformation_expr(0); - setState(39); + setState(43); + transformation_expr(); + setState(44); match(RPAREN); } break; case 3: + _localctx = new ArithExpressionContext(_localctx); + enterOuterAlt(_localctx, 3); { - _localctx = new TransformationEntityContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; - setState(41); - transformation_entity(); + setState(46); + arithmetic_expr(0); } break; case 4: + _localctx = new TransformationEntityContext(_localctx); + enterOuterAlt(_localctx, 4); { - _localctx = new NotFuncContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; - setState(42); - match(NOT); - setState(43); - match(LPAREN); - setState(44); - transformation_expr(0); - setState(45); - match(RPAREN); + setState(47); + transformation_entity(); } break; case 5: + _localctx = new ComparisonExpressionContext(_localctx); + enterOuterAlt(_localctx, 5); { - _localctx = new TernaryFuncContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; - setState(47); - match(IF); setState(48); - transformation_expr(0); - setState(49); - match(QUESTION); - setState(50); - transformation_expr(0); - setState(51); - match(COLON); - setState(52); - transformation_expr(0); + comparison_expr(0); } break; - case 6: + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class Conditional_exprContext extends ParserRuleContext { + public Conditional_exprContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_conditional_expr; } + + public Conditional_exprContext() { } + public void copyFrom(Conditional_exprContext ctx) { + super.copyFrom(ctx); + } + } + public static class TernaryFuncWithoutIfContext extends Conditional_exprContext { + public Comparison_exprContext comparison_expr() { + return getRuleContext(Comparison_exprContext.class,0); + } + public TerminalNode QUESTION() { return getToken(StellarParser.QUESTION, 0); } + public List transformation_expr() { + return getRuleContexts(Transformation_exprContext.class); + } + public Transformation_exprContext transformation_expr(int i) { + return getRuleContext(Transformation_exprContext.class,i); + } + public TerminalNode COLON() { return getToken(StellarParser.COLON, 0); } + public TernaryFuncWithoutIfContext(Conditional_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterTernaryFuncWithoutIf(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitTernaryFuncWithoutIf(this); + } + } + public static class TernaryFuncWithIfContext extends Conditional_exprContext { + public TerminalNode IF() { return getToken(StellarParser.IF, 0); } + public Comparison_exprContext comparison_expr() { + return getRuleContext(Comparison_exprContext.class,0); + } + public TerminalNode THEN() { return getToken(StellarParser.THEN, 0); } + public List transformation_expr() { + return getRuleContexts(Transformation_exprContext.class); + } + public Transformation_exprContext transformation_expr(int i) { + return getRuleContext(Transformation_exprContext.class,i); + } + public TerminalNode ELSE() { return getToken(StellarParser.ELSE, 0); } + public TernaryFuncWithIfContext(Conditional_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterTernaryFuncWithIf(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitTernaryFuncWithIf(this); + } + } + + public final Conditional_exprContext conditional_expr() throws RecognitionException { + Conditional_exprContext _localctx = new Conditional_exprContext(_ctx, getState()); + enterRule(_localctx, 4, RULE_conditional_expr); + try { + setState(64); + switch (_input.LA(1)) { + case NOT: + case TRUE: + case FALSE: + case NULL: + case LBRACE: + case LBRACKET: + case LPAREN: + case EXISTS: + case INT_LITERAL: + case DOUBLE_LITERAL: + case IDENTIFIER: + case STRING_LITERAL: + _localctx = new TernaryFuncWithoutIfContext(_localctx); + enterOuterAlt(_localctx, 1); { - _localctx = new ArithExpressionContext(_localctx); - _ctx = _localctx; - _prevctx = _localctx; + setState(51); + comparison_expr(0); + setState(52); + match(QUESTION); + setState(53); + transformation_expr(); setState(54); - arithmetic_expr(0); + match(COLON); + setState(55); + transformation_expr(); } break; - } - _ctx.stop = _input.LT(-1); - setState(71); - _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,2,_ctx); - while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { - if ( _alt==1 ) { - if ( _parseListeners!=null ) triggerExitRuleEvent(); - _prevctx = _localctx; - { - setState(69); - switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { - case 1: - { - _localctx = new LogicalExpressionAndContext(new Transformation_exprContext(_parentctx, _parentState)); - pushNewRecursionContext(_localctx, _startState, RULE_transformation_expr); - setState(57); - if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)"); - setState(58); - match(AND); - setState(59); - transformation_expr(7); - } - break; - case 2: - { - _localctx = new LogicalExpressionOrContext(new Transformation_exprContext(_parentctx, _parentState)); - pushNewRecursionContext(_localctx, _startState, RULE_transformation_expr); - setState(60); - if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(61); - match(OR); - setState(62); - transformation_expr(6); - } - break; - case 3: - { - _localctx = new TernaryFuncContext(new Transformation_exprContext(_parentctx, _parentState)); - pushNewRecursionContext(_localctx, _startState, RULE_transformation_expr); - setState(63); - if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(64); - match(QUESTION); - setState(65); - transformation_expr(0); - setState(66); - match(COLON); - setState(67); - transformation_expr(4); - } - break; - } - } + case IF: + _localctx = new TernaryFuncWithIfContext(_localctx); + enterOuterAlt(_localctx, 2); + { + setState(57); + match(IF); + setState(58); + comparison_expr(0); + setState(59); + match(THEN); + setState(60); + transformation_expr(); + setState(61); + match(ELSE); + setState(62); + transformation_expr(); } - setState(73); - _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,2,_ctx); - } + break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -460,7 +425,7 @@ private Transformation_exprContext transformation_expr(int _p) throws Recognitio _errHandler.recover(this, re); } finally { - unrollRecursionContexts(_parentctx); + exitRule(); } return _localctx; } @@ -476,6 +441,23 @@ public void copyFrom(Comparison_exprContext ctx) { super.copyFrom(ctx); } } + public static class NotFuncContext extends Comparison_exprContext { + public TerminalNode NOT() { return getToken(StellarParser.NOT, 0); } + public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } + public Comparison_exprContext comparison_expr() { + return getRuleContext(Comparison_exprContext.class,0); + } + public TerminalNode RPAREN() { return getToken(StellarParser.RPAREN, 0); } + public NotFuncContext(Comparison_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterNotFunc(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitNotFunc(this); + } + } public static class ComparisonExpressionParensContext extends Comparison_exprContext { public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } public Comparison_exprContext comparison_expr() { @@ -530,6 +512,24 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitComparisonExpressionWithOperator(this); } } + public static class LogicalExpressionAndContext extends Comparison_exprContext { + public List comparison_expr() { + return getRuleContexts(Comparison_exprContext.class); + } + public Comparison_exprContext comparison_expr(int i) { + return getRuleContext(Comparison_exprContext.class,i); + } + public TerminalNode AND() { return getToken(StellarParser.AND, 0); } + public LogicalExpressionAndContext(Comparison_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterLogicalExpressionAnd(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitLogicalExpressionAnd(this); + } + } public static class NInExpressionContext extends Comparison_exprContext { public List identifier_operand() { return getRuleContexts(Identifier_operandContext.class); @@ -548,61 +548,176 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitNInExpression(this); } } + public static class LogicalExpressionOrContext extends Comparison_exprContext { + public List comparison_expr() { + return getRuleContexts(Comparison_exprContext.class); + } + public Comparison_exprContext comparison_expr(int i) { + return getRuleContext(Comparison_exprContext.class,i); + } + public TerminalNode OR() { return getToken(StellarParser.OR, 0); } + public LogicalExpressionOrContext(Comparison_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterLogicalExpressionOr(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitLogicalExpressionOr(this); + } + } + public static class OperandContext extends Comparison_exprContext { + public Identifier_operandContext identifier_operand() { + return getRuleContext(Identifier_operandContext.class,0); + } + public OperandContext(Comparison_exprContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterOperand(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitOperand(this); + } + } public final Comparison_exprContext comparison_expr() throws RecognitionException { - Comparison_exprContext _localctx = new Comparison_exprContext(_ctx, getState()); - enterRule(_localctx, 4, RULE_comparison_expr); + return comparison_expr(0); + } + + private Comparison_exprContext comparison_expr(int _p) throws RecognitionException { + ParserRuleContext _parentctx = _ctx; + int _parentState = getState(); + Comparison_exprContext _localctx = new Comparison_exprContext(_ctx, _parentState); + Comparison_exprContext _prevctx = _localctx; + int _startState = 6; + enterRecursionRule(_localctx, 6, RULE_comparison_expr, _p); try { - setState(90); - switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(89); + switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: - _localctx = new ComparisonExpressionWithOperatorContext(_localctx); - enterOuterAlt(_localctx, 1); { - setState(74); + _localctx = new ComparisonExpressionWithOperatorContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + + setState(67); comparison_operand(); - setState(75); + setState(68); comp_operator(); - setState(76); + setState(69); comparison_operand(); } break; case 2: - _localctx = new InExpressionContext(_localctx); - enterOuterAlt(_localctx, 2); { - setState(78); + _localctx = new InExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(71); identifier_operand(); - setState(79); + setState(72); match(IN); - setState(80); + setState(73); identifier_operand(); } break; case 3: - _localctx = new NInExpressionContext(_localctx); - enterOuterAlt(_localctx, 3); { - setState(82); + _localctx = new NInExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(75); identifier_operand(); - setState(83); + setState(76); match(NIN); - setState(84); + setState(77); identifier_operand(); } break; case 4: - _localctx = new ComparisonExpressionParensContext(_localctx); - enterOuterAlt(_localctx, 4); { - setState(86); + _localctx = new NotFuncContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(79); + match(NOT); + setState(80); match(LPAREN); - setState(87); - comparison_expr(); - setState(88); + setState(81); + comparison_expr(0); + setState(82); + match(RPAREN); + } + break; + case 5: + { + _localctx = new ComparisonExpressionParensContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(84); + match(LPAREN); + setState(85); + comparison_expr(0); + setState(86); match(RPAREN); } break; + case 6: + { + _localctx = new OperandContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(88); + identifier_operand(); + } + break; + } + _ctx.stop = _input.LT(-1); + setState(99); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + if ( _parseListeners!=null ) triggerExitRuleEvent(); + _prevctx = _localctx; + { + setState(97); + switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) { + case 1: + { + _localctx = new LogicalExpressionAndContext(new Comparison_exprContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_comparison_expr); + setState(91); + if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); + setState(92); + match(AND); + setState(93); + comparison_expr(6); + } + break; + case 2: + { + _localctx = new LogicalExpressionOrContext(new Comparison_exprContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_comparison_expr); + setState(94); + if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); + setState(95); + match(OR); + setState(96); + comparison_expr(5); + } + break; + } + } + } + setState(101); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + } } } catch (RecognitionException re) { @@ -611,7 +726,7 @@ public final Comparison_exprContext comparison_expr() throws RecognitionExceptio _errHandler.recover(this, re); } finally { - exitRule(); + unrollRecursionContexts(_parentctx); } return _localctx; } @@ -644,12 +759,12 @@ public void exitRule(ParseTreeListener listener) { public final Comparison_operandContext comparison_operand() throws RecognitionException { Comparison_operandContext _localctx = new Comparison_operandContext(_ctx, getState()); - enterRule(_localctx, 6, RULE_comparison_operand); + enterRule(_localctx, 8, RULE_comparison_operand); try { _localctx = new IdentifierOperandContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(92); + setState(102); identifier_operand(); } } @@ -684,11 +799,11 @@ public void exitRule(ParseTreeListener listener) { public final Transformation_entityContext transformation_entity() throws RecognitionException { Transformation_entityContext _localctx = new Transformation_entityContext(_ctx, getState()); - enterRule(_localctx, 8, RULE_transformation_entity); + enterRule(_localctx, 10, RULE_transformation_entity); try { enterOuterAlt(_localctx, 1); { - setState(94); + setState(104); identifier_operand(); } } @@ -734,13 +849,13 @@ public void exitRule(ParseTreeListener listener) { public final Comp_operatorContext comp_operator() throws RecognitionException { Comp_operatorContext _localctx = new Comp_operatorContext(_ctx, getState()); - enterRule(_localctx, 10, RULE_comp_operator); + enterRule(_localctx, 12, RULE_comp_operator); int _la; try { _localctx = new ComparisonOpContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(96); + setState(106); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << NEQ) | (1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) { _errHandler.recoverInline(this); @@ -787,13 +902,13 @@ public void exitRule(ParseTreeListener listener) { public final Arith_operator_additionContext arith_operator_addition() throws RecognitionException { Arith_operator_additionContext _localctx = new Arith_operator_additionContext(_ctx, getState()); - enterRule(_localctx, 12, RULE_arith_operator_addition); + enterRule(_localctx, 14, RULE_arith_operator_addition); int _la; try { _localctx = new ArithOp_plusContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(98); + setState(108); _la = _input.LA(1); if ( !(_la==MINUS || _la==PLUS) ) { _errHandler.recoverInline(this); @@ -840,13 +955,13 @@ public void exitRule(ParseTreeListener listener) { public final Arith_operator_mulContext arith_operator_mul() throws RecognitionException { Arith_operator_mulContext _localctx = new Arith_operator_mulContext(_ctx, getState()); - enterRule(_localctx, 14, RULE_arith_operator_mul); + enterRule(_localctx, 16, RULE_arith_operator_mul); int _la; try { _localctx = new ArithOp_mulContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(100); + setState(110); _la = _input.LA(1); if ( !(_la==DIV || _la==MUL) ) { _errHandler.recoverInline(this); @@ -867,9 +982,11 @@ public final Arith_operator_mulContext arith_operator_mul() throws RecognitionEx } public static class Func_argsContext extends ParserRuleContext { + public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } public Op_listContext op_list() { return getRuleContext(Op_listContext.class,0); } + public TerminalNode RPAREN() { return getToken(StellarParser.RPAREN, 0); } public Func_argsContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @@ -886,12 +1003,30 @@ public void exitRule(ParseTreeListener listener) { public final Func_argsContext func_args() throws RecognitionException { Func_argsContext _localctx = new Func_argsContext(_ctx, getState()); - enterRule(_localctx, 16, RULE_func_args); + enterRule(_localctx, 18, RULE_func_args); try { - enterOuterAlt(_localctx, 1); - { - setState(102); - op_list(0); + setState(118); + switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(112); + match(LPAREN); + setState(113); + op_list(0); + setState(114); + match(RPAREN); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(116); + match(LPAREN); + setState(117); + match(RPAREN); + } + break; } } catch (RecognitionException re) { @@ -936,20 +1071,20 @@ private Op_listContext op_list(int _p) throws RecognitionException { int _parentState = getState(); Op_listContext _localctx = new Op_listContext(_ctx, _parentState); Op_listContext _prevctx = _localctx; - int _startState = 18; - enterRecursionRule(_localctx, 18, RULE_op_list, _p); + int _startState = 20; + enterRecursionRule(_localctx, 20, RULE_op_list, _p); try { int _alt; enterOuterAlt(_localctx, 1); { { - setState(105); + setState(121); identifier_operand(); } _ctx.stop = _input.LT(-1); - setState(112); + setState(128); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + _alt = getInterpreter().adaptivePredict(_input,6,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); @@ -958,18 +1093,18 @@ private Op_listContext op_list(int _p) throws RecognitionException { { _localctx = new Op_listContext(_parentctx, _parentState); pushNewRecursionContext(_localctx, _startState, RULE_op_list); - setState(107); + setState(123); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(108); + setState(124); match(COMMA); - setState(109); + setState(125); identifier_operand(); } } } - setState(114); + setState(130); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + _alt = getInterpreter().adaptivePredict(_input,6,_ctx); } } } @@ -1006,27 +1141,27 @@ public void exitRule(ParseTreeListener listener) { public final List_entityContext list_entity() throws RecognitionException { List_entityContext _localctx = new List_entityContext(_ctx, getState()); - enterRule(_localctx, 20, RULE_list_entity); + enterRule(_localctx, 22, RULE_list_entity); try { - setState(121); - switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { + setState(137); + switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(115); + setState(131); match(LBRACKET); - setState(116); + setState(132); op_list(0); - setState(117); + setState(133); match(RBRACKET); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(119); + setState(135); match(LBRACKET); - setState(120); + setState(136); match(RBRACKET); } break; @@ -1043,6 +1178,156 @@ public final List_entityContext list_entity() throws RecognitionException { return _localctx; } + public static class Kv_listContext extends ParserRuleContext { + public Identifier_operandContext identifier_operand() { + return getRuleContext(Identifier_operandContext.class,0); + } + public TerminalNode COLON() { return getToken(StellarParser.COLON, 0); } + public Transformation_exprContext transformation_expr() { + return getRuleContext(Transformation_exprContext.class,0); + } + public Kv_listContext kv_list() { + return getRuleContext(Kv_listContext.class,0); + } + public TerminalNode COMMA() { return getToken(StellarParser.COMMA, 0); } + public Kv_listContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_kv_list; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterKv_list(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitKv_list(this); + } + } + + public final Kv_listContext kv_list() throws RecognitionException { + return kv_list(0); + } + + private Kv_listContext kv_list(int _p) throws RecognitionException { + ParserRuleContext _parentctx = _ctx; + int _parentState = getState(); + Kv_listContext _localctx = new Kv_listContext(_ctx, _parentState); + Kv_listContext _prevctx = _localctx; + int _startState = 24; + enterRecursionRule(_localctx, 24, RULE_kv_list, _p); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + { + setState(140); + identifier_operand(); + setState(141); + match(COLON); + setState(142); + transformation_expr(); + } + _ctx.stop = _input.LT(-1); + setState(152); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,8,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + if ( _parseListeners!=null ) triggerExitRuleEvent(); + _prevctx = _localctx; + { + { + _localctx = new Kv_listContext(_parentctx, _parentState); + pushNewRecursionContext(_localctx, _startState, RULE_kv_list); + setState(144); + if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); + setState(145); + match(COMMA); + setState(146); + identifier_operand(); + setState(147); + match(COLON); + setState(148); + transformation_expr(); + } + } + } + setState(154); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,8,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + unrollRecursionContexts(_parentctx); + } + return _localctx; + } + + public static class Map_entityContext extends ParserRuleContext { + public TerminalNode LBRACE() { return getToken(StellarParser.LBRACE, 0); } + public Kv_listContext kv_list() { + return getRuleContext(Kv_listContext.class,0); + } + public TerminalNode RBRACE() { return getToken(StellarParser.RBRACE, 0); } + public Map_entityContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_map_entity; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterMap_entity(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitMap_entity(this); + } + } + + public final Map_entityContext map_entity() throws RecognitionException { + Map_entityContext _localctx = new Map_entityContext(_ctx, getState()); + enterRule(_localctx, 26, RULE_map_entity); + try { + setState(161); + switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(155); + match(LBRACE); + setState(156); + kv_list(0); + setState(157); + match(RBRACE); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(159); + match(LBRACE); + setState(160); + match(RBRACE); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + public static class Arithmetic_exprContext extends ParserRuleContext { public Arithmetic_exprContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -1114,8 +1399,8 @@ private Arithmetic_exprContext arithmetic_expr(int _p) throws RecognitionExcepti int _parentState = getState(); Arithmetic_exprContext _localctx = new Arithmetic_exprContext(_ctx, _parentState); Arithmetic_exprContext _prevctx = _localctx; - int _startState = 22; - enterRecursionRule(_localctx, 22, RULE_arithmetic_expr, _p); + int _startState = 28; + enterRecursionRule(_localctx, 28, RULE_arithmetic_expr, _p); try { int _alt; enterOuterAlt(_localctx, 1); @@ -1125,29 +1410,29 @@ private Arithmetic_exprContext arithmetic_expr(int _p) throws RecognitionExcepti _ctx = _localctx; _prevctx = _localctx; - setState(124); + setState(164); arithmetic_expr_mul(0); } _ctx.stop = _input.LT(-1); - setState(134); + setState(174); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,7,_ctx); + _alt = getInterpreter().adaptivePredict(_input,11,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(132); - switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { + setState(172); + switch ( getInterpreter().adaptivePredict(_input,10,_ctx) ) { case 1: { _localctx = new ArithExpr_plusContext(new Arithmetic_exprContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_arithmetic_expr); - setState(126); + setState(166); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(127); + setState(167); match(PLUS); - setState(128); + setState(168); arithmetic_expr_mul(0); } break; @@ -1155,20 +1440,20 @@ private Arithmetic_exprContext arithmetic_expr(int _p) throws RecognitionExcepti { _localctx = new ArithExpr_minusContext(new Arithmetic_exprContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_arithmetic_expr); - setState(129); + setState(169); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(130); + setState(170); match(MINUS); - setState(131); + setState(171); arithmetic_expr_mul(0); } break; } } } - setState(136); + setState(176); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,7,_ctx); + _alt = getInterpreter().adaptivePredict(_input,11,_ctx); } } } @@ -1254,8 +1539,8 @@ private Arithmetic_expr_mulContext arithmetic_expr_mul(int _p) throws Recognitio int _parentState = getState(); Arithmetic_expr_mulContext _localctx = new Arithmetic_expr_mulContext(_ctx, _parentState); Arithmetic_expr_mulContext _prevctx = _localctx; - int _startState = 24; - enterRecursionRule(_localctx, 24, RULE_arithmetic_expr_mul, _p); + int _startState = 30; + enterRecursionRule(_localctx, 30, RULE_arithmetic_expr_mul, _p); try { int _alt; enterOuterAlt(_localctx, 1); @@ -1265,29 +1550,29 @@ private Arithmetic_expr_mulContext arithmetic_expr_mul(int _p) throws Recognitio _ctx = _localctx; _prevctx = _localctx; - setState(138); + setState(178); arithmetic_operands(); } _ctx.stop = _input.LT(-1); - setState(148); + setState(188); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,9,_ctx); + _alt = getInterpreter().adaptivePredict(_input,13,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(146); - switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) { + setState(186); + switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: { _localctx = new ArithExpr_mulContext(new Arithmetic_expr_mulContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_arithmetic_expr_mul); - setState(140); + setState(180); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(141); + setState(181); match(MUL); - setState(142); + setState(182); arithmetic_expr_mul(3); } break; @@ -1295,20 +1580,20 @@ private Arithmetic_expr_mulContext arithmetic_expr_mul(int _p) throws Recognitio { _localctx = new ArithExpr_divContext(new Arithmetic_expr_mulContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_arithmetic_expr_mul); - setState(143); + setState(183); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(144); + setState(184); match(DIV); - setState(145); + setState(185); arithmetic_expr_mul(2); } break; } } } - setState(150); + setState(190); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,9,_ctx); + _alt = getInterpreter().adaptivePredict(_input,13,_ctx); } } } @@ -1336,11 +1621,9 @@ public void copyFrom(FunctionsContext ctx) { } public static class TransformationFuncContext extends FunctionsContext { public TerminalNode IDENTIFIER() { return getToken(StellarParser.IDENTIFIER, 0); } - public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } public Func_argsContext func_args() { return getRuleContext(Func_argsContext.class,0); } - public TerminalNode RPAREN() { return getToken(StellarParser.RPAREN, 0); } public TransformationFuncContext(FunctionsContext ctx) { copyFrom(ctx); } @Override public void enterRule(ParseTreeListener listener) { @@ -1354,19 +1637,15 @@ public void exitRule(ParseTreeListener listener) { public final FunctionsContext functions() throws RecognitionException { FunctionsContext _localctx = new FunctionsContext(_ctx, getState()); - enterRule(_localctx, 26, RULE_functions); + enterRule(_localctx, 32, RULE_functions); try { _localctx = new TransformationFuncContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(151); + setState(191); match(IDENTIFIER); - setState(152); - match(LPAREN); - setState(153); + setState(192); func_args(); - setState(154); - match(RPAREN); } } catch (RecognitionException re) { @@ -1460,15 +1739,15 @@ public void exitRule(ParseTreeListener listener) { public final Arithmetic_operandsContext arithmetic_operands() throws RecognitionException { Arithmetic_operandsContext _localctx = new Arithmetic_operandsContext(_ctx, getState()); - enterRule(_localctx, 28, RULE_arithmetic_operands); + enterRule(_localctx, 34, RULE_arithmetic_operands); try { - setState(164); - switch ( getInterpreter().adaptivePredict(_input,10,_ctx) ) { + setState(202); + switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: _localctx = new NumericFunctionsContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(156); + setState(194); functions(); } break; @@ -1476,7 +1755,7 @@ public final Arithmetic_operandsContext arithmetic_operands() throws Recognition _localctx = new DoubleLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(157); + setState(195); match(DOUBLE_LITERAL); } break; @@ -1484,7 +1763,7 @@ public final Arithmetic_operandsContext arithmetic_operands() throws Recognition _localctx = new IntLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(158); + setState(196); match(INT_LITERAL); } break; @@ -1492,7 +1771,7 @@ public final Arithmetic_operandsContext arithmetic_operands() throws Recognition _localctx = new VariableContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(159); + setState(197); match(IDENTIFIER); } break; @@ -1500,11 +1779,11 @@ public final Arithmetic_operandsContext arithmetic_operands() throws Recognition _localctx = new ParenArithContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(160); + setState(198); match(LPAREN); - setState(161); + setState(199); arithmetic_expr(0); - setState(162); + setState(200); match(RPAREN); } break; @@ -1572,6 +1851,20 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitList(this); } } + public static class MapConstContext extends Identifier_operandContext { + public Map_entityContext map_entity() { + return getRuleContext(Map_entityContext.class,0); + } + public MapConstContext(Identifier_operandContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterMapConst(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitMapConst(this); + } + } public static class LogicalConstContext extends Identifier_operandContext { public TerminalNode TRUE() { return getToken(StellarParser.TRUE, 0); } public TerminalNode FALSE() { return getToken(StellarParser.FALSE, 0); } @@ -1585,6 +1878,18 @@ public void exitRule(ParseTreeListener listener) { if ( listener instanceof StellarListener ) ((StellarListener)listener).exitLogicalConst(this); } } + public static class NullConstContext extends Identifier_operandContext { + public TerminalNode NULL() { return getToken(StellarParser.NULL, 0); } + public NullConstContext(Identifier_operandContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).enterNullConst(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof StellarListener ) ((StellarListener)listener).exitNullConst(this); + } + } public static class ExistsFuncContext extends Identifier_operandContext { public TerminalNode EXISTS() { return getToken(StellarParser.EXISTS, 0); } public TerminalNode LPAREN() { return getToken(StellarParser.LPAREN, 0); } @@ -1603,17 +1908,17 @@ public void exitRule(ParseTreeListener listener) { public final Identifier_operandContext identifier_operand() throws RecognitionException { Identifier_operandContext _localctx = new Identifier_operandContext(_ctx, getState()); - enterRule(_localctx, 30, RULE_identifier_operand); + enterRule(_localctx, 36, RULE_identifier_operand); int _la; try { - setState(174); + setState(214); switch (_input.LA(1)) { case TRUE: case FALSE: _localctx = new LogicalConstContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(166); + setState(204); _la = _input.LA(1); if ( !(_la==TRUE || _la==FALSE) ) { _errHandler.recoverInline(this); @@ -1629,7 +1934,7 @@ public final Identifier_operandContext identifier_operand() throws RecognitionEx _localctx = new ArithmeticOperandsContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(167); + setState(205); arithmetic_expr(0); } break; @@ -1637,7 +1942,7 @@ public final Identifier_operandContext identifier_operand() throws RecognitionEx _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(168); + setState(206); match(STRING_LITERAL); } break; @@ -1645,21 +1950,37 @@ public final Identifier_operandContext identifier_operand() throws RecognitionEx _localctx = new ListContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(169); + setState(207); list_entity(); } break; + case LBRACE: + _localctx = new MapConstContext(_localctx); + enterOuterAlt(_localctx, 5); + { + setState(208); + map_entity(); + } + break; + case NULL: + _localctx = new NullConstContext(_localctx); + enterOuterAlt(_localctx, 6); + { + setState(209); + match(NULL); + } + break; case EXISTS: _localctx = new ExistsFuncContext(_localctx); - enterOuterAlt(_localctx, 5); + enterOuterAlt(_localctx, 7); { - setState(170); + setState(210); match(EXISTS); - setState(171); + setState(211); match(LPAREN); - setState(172); + setState(212); match(IDENTIFIER); - setState(173); + setState(213); match(RPAREN); } break; @@ -1680,29 +2001,36 @@ public final Identifier_operandContext identifier_operand() throws RecognitionEx public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 1: - return transformation_expr_sempred((Transformation_exprContext)_localctx, predIndex); - case 9: + case 3: + return comparison_expr_sempred((Comparison_exprContext)_localctx, predIndex); + case 10: return op_list_sempred((Op_listContext)_localctx, predIndex); - case 11: - return arithmetic_expr_sempred((Arithmetic_exprContext)_localctx, predIndex); case 12: + return kv_list_sempred((Kv_listContext)_localctx, predIndex); + case 14: + return arithmetic_expr_sempred((Arithmetic_exprContext)_localctx, predIndex); + case 15: return arithmetic_expr_mul_sempred((Arithmetic_expr_mulContext)_localctx, predIndex); } return true; } - private boolean transformation_expr_sempred(Transformation_exprContext _localctx, int predIndex) { + private boolean comparison_expr_sempred(Comparison_exprContext _localctx, int predIndex) { switch (predIndex) { case 0: - return precpred(_ctx, 6); - case 1: return precpred(_ctx, 5); - case 2: - return precpred(_ctx, 3); + case 1: + return precpred(_ctx, 4); } return true; } private boolean op_list_sempred(Op_listContext _localctx, int predIndex) { + switch (predIndex) { + case 2: + return precpred(_ctx, 1); + } + return true; + } + private boolean kv_list_sempred(Kv_listContext _localctx, int predIndex) { switch (predIndex) { case 3: return precpred(_ctx, 1); @@ -1729,59 +2057,75 @@ private boolean arithmetic_expr_mul_sempred(Arithmetic_expr_mulContext _localctx } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3\"\u00b3\4\2\t\2\4"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3\'\u00db\4\2\t\2\4"+ "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+ - "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\3\2\3\2\3"+ - "\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+ - "\3\3\3\3\3\3\5\3:\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+ - "\7\3H\n\3\f\3\16\3K\13\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3"+ - "\4\3\4\3\4\3\4\3\4\5\4]\n\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3"+ - "\n\3\n\3\13\3\13\3\13\3\13\3\13\3\13\7\13q\n\13\f\13\16\13t\13\13\3\f"+ - "\3\f\3\f\3\f\3\f\3\f\5\f|\n\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\7\r"+ - "\u0087\n\r\f\r\16\r\u008a\13\r\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+ - "\3\16\7\16\u0095\n\16\f\16\16\16\u0098\13\16\3\17\3\17\3\17\3\17\3\17"+ - "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00a7\n\20\3\21\3\21\3\21"+ - "\3\21\3\21\3\21\3\21\3\21\5\21\u00b1\n\21\3\21\2\6\4\24\30\32\22\2\4\6"+ - "\b\n\f\16\20\22\24\26\30\32\34\36 \2\6\3\2\t\16\3\2\22\23\3\2\24\25\3"+ - "\2\7\b\u00bb\2\"\3\2\2\2\49\3\2\2\2\6\\\3\2\2\2\b^\3\2\2\2\n`\3\2\2\2"+ - "\fb\3\2\2\2\16d\3\2\2\2\20f\3\2\2\2\22h\3\2\2\2\24j\3\2\2\2\26{\3\2\2"+ - "\2\30}\3\2\2\2\32\u008b\3\2\2\2\34\u0099\3\2\2\2\36\u00a6\3\2\2\2 \u00b0"+ - "\3\2\2\2\"#\5\4\3\2#$\7\2\2\3$\3\3\2\2\2%&\b\3\1\2&:\5\6\4\2\'(\7\30\2"+ - "\2()\5\4\3\2)*\7\31\2\2*:\3\2\2\2+:\5\n\6\2,-\7\6\2\2-.\7\30\2\2./\5\4"+ - "\3\2/\60\7\31\2\2\60:\3\2\2\2\61\62\7\21\2\2\62\63\5\4\3\2\63\64\7\17"+ - "\2\2\64\65\5\4\3\2\65\66\7\20\2\2\66\67\5\4\3\2\67:\3\2\2\28:\5\30\r\2"+ - "9%\3\2\2\29\'\3\2\2\29+\3\2\2\29,\3\2\2\29\61\3\2\2\298\3\2\2\2:I\3\2"+ - "\2\2;<\f\b\2\2<=\7\4\2\2=H\5\4\3\t>?\f\7\2\2?@\7\5\2\2@H\5\4\3\bAB\f\5"+ - "\2\2BC\7\17\2\2CD\5\4\3\2DE\7\20\2\2EF\5\4\3\6FH\3\2\2\2G;\3\2\2\2G>\3"+ - "\2\2\2GA\3\2\2\2HK\3\2\2\2IG\3\2\2\2IJ\3\2\2\2J\5\3\2\2\2KI\3\2\2\2LM"+ - "\5\b\5\2MN\5\f\7\2NO\5\b\5\2O]\3\2\2\2PQ\5 \21\2QR\7\32\2\2RS\5 \21\2"+ - "S]\3\2\2\2TU\5 \21\2UV\7\33\2\2VW\5 \21\2W]\3\2\2\2XY\7\30\2\2YZ\5\6\4"+ - "\2Z[\7\31\2\2[]\3\2\2\2\\L\3\2\2\2\\P\3\2\2\2\\T\3\2\2\2\\X\3\2\2\2]\7"+ - "\3\2\2\2^_\5 \21\2_\t\3\2\2\2`a\5 \21\2a\13\3\2\2\2bc\t\2\2\2c\r\3\2\2"+ - "\2de\t\3\2\2e\17\3\2\2\2fg\t\4\2\2g\21\3\2\2\2hi\5\24\13\2i\23\3\2\2\2"+ - "jk\b\13\1\2kl\5 \21\2lr\3\2\2\2mn\f\3\2\2no\7\3\2\2oq\5 \21\2pm\3\2\2"+ - "\2qt\3\2\2\2rp\3\2\2\2rs\3\2\2\2s\25\3\2\2\2tr\3\2\2\2uv\7\26\2\2vw\5"+ - "\24\13\2wx\7\27\2\2x|\3\2\2\2yz\7\26\2\2z|\7\27\2\2{u\3\2\2\2{y\3\2\2"+ - "\2|\27\3\2\2\2}~\b\r\1\2~\177\5\32\16\2\177\u0088\3\2\2\2\u0080\u0081"+ - "\f\4\2\2\u0081\u0082\7\23\2\2\u0082\u0087\5\32\16\2\u0083\u0084\f\3\2"+ - "\2\u0084\u0085\7\22\2\2\u0085\u0087\5\32\16\2\u0086\u0080\3\2\2\2\u0086"+ - "\u0083\3\2\2\2\u0087\u008a\3\2\2\2\u0088\u0086\3\2\2\2\u0088\u0089\3\2"+ - "\2\2\u0089\31\3\2\2\2\u008a\u0088\3\2\2\2\u008b\u008c\b\16\1\2\u008c\u008d"+ - "\5\36\20\2\u008d\u0096\3\2\2\2\u008e\u008f\f\4\2\2\u008f\u0090\7\25\2"+ - "\2\u0090\u0095\5\32\16\5\u0091\u0092\f\3\2\2\u0092\u0093\7\24\2\2\u0093"+ - "\u0095\5\32\16\4\u0094\u008e\3\2\2\2\u0094\u0091\3\2\2\2\u0095\u0098\3"+ - "\2\2\2\u0096\u0094\3\2\2\2\u0096\u0097\3\2\2\2\u0097\33\3\2\2\2\u0098"+ - "\u0096\3\2\2\2\u0099\u009a\7\37\2\2\u009a\u009b\7\30\2\2\u009b\u009c\5"+ - "\22\n\2\u009c\u009d\7\31\2\2\u009d\35\3\2\2\2\u009e\u00a7\5\34\17\2\u009f"+ - "\u00a7\7\36\2\2\u00a0\u00a7\7\35\2\2\u00a1\u00a7\7\37\2\2\u00a2\u00a3"+ - "\7\30\2\2\u00a3\u00a4\5\30\r\2\u00a4\u00a5\7\31\2\2\u00a5\u00a7\3\2\2"+ - "\2\u00a6\u009e\3\2\2\2\u00a6\u009f\3\2\2\2\u00a6\u00a0\3\2\2\2\u00a6\u00a1"+ - "\3\2\2\2\u00a6\u00a2\3\2\2\2\u00a7\37\3\2\2\2\u00a8\u00b1\t\5\2\2\u00a9"+ - "\u00b1\5\30\r\2\u00aa\u00b1\7 \2\2\u00ab\u00b1\5\26\f\2\u00ac\u00ad\7"+ - "\34\2\2\u00ad\u00ae\7\30\2\2\u00ae\u00af\7\37\2\2\u00af\u00b1\7\31\2\2"+ - "\u00b0\u00a8\3\2\2\2\u00b0\u00a9\3\2\2\2\u00b0\u00aa\3\2\2\2\u00b0\u00ab"+ - "\3\2\2\2\u00b0\u00ac\3\2\2\2\u00b1!\3\2\2\2\169GI\\r{\u0086\u0088\u0094"+ - "\u0096\u00a6\u00b0"; + "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ + "\4\23\t\23\4\24\t\24\3\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3\64"+ + "\n\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\5\4C\n\4\3\5"+ + "\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+ + "\5\3\5\3\5\3\5\3\5\5\5\\\n\5\3\5\3\5\3\5\3\5\3\5\3\5\7\5d\n\5\f\5\16\5"+ + "g\13\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\13\3\13\3\13\3\13\3\13"+ + "\3\13\5\13y\n\13\3\f\3\f\3\f\3\f\3\f\3\f\7\f\u0081\n\f\f\f\16\f\u0084"+ + "\13\f\3\r\3\r\3\r\3\r\3\r\3\r\5\r\u008c\n\r\3\16\3\16\3\16\3\16\3\16\3"+ + "\16\3\16\3\16\3\16\3\16\3\16\7\16\u0099\n\16\f\16\16\16\u009c\13\16\3"+ + "\17\3\17\3\17\3\17\3\17\3\17\5\17\u00a4\n\17\3\20\3\20\3\20\3\20\3\20"+ + "\3\20\3\20\3\20\3\20\7\20\u00af\n\20\f\20\16\20\u00b2\13\20\3\21\3\21"+ + "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\7\21\u00bd\n\21\f\21\16\21\u00c0\13"+ + "\21\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u00cd"+ + "\n\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\5\24\u00d9\n\24"+ + "\3\24\2\7\b\26\32\36 \25\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&\2"+ + "\6\3\2\t\16\3\2\25\26\3\2\27\30\3\2\7\b\u00e6\2(\3\2\2\2\4\63\3\2\2\2"+ + "\6B\3\2\2\2\b[\3\2\2\2\nh\3\2\2\2\fj\3\2\2\2\16l\3\2\2\2\20n\3\2\2\2\22"+ + "p\3\2\2\2\24x\3\2\2\2\26z\3\2\2\2\30\u008b\3\2\2\2\32\u008d\3\2\2\2\34"+ + "\u00a3\3\2\2\2\36\u00a5\3\2\2\2 \u00b3\3\2\2\2\"\u00c1\3\2\2\2$\u00cc"+ + "\3\2\2\2&\u00d8\3\2\2\2()\5\4\3\2)*\7\2\2\3*\3\3\2\2\2+\64\5\6\4\2,-\7"+ + "\35\2\2-.\5\4\3\2./\7\36\2\2/\64\3\2\2\2\60\64\5\36\20\2\61\64\5\f\7\2"+ + "\62\64\5\b\5\2\63+\3\2\2\2\63,\3\2\2\2\63\60\3\2\2\2\63\61\3\2\2\2\63"+ + "\62\3\2\2\2\64\5\3\2\2\2\65\66\5\b\5\2\66\67\7\17\2\2\678\5\4\3\289\7"+ + "\20\2\29:\5\4\3\2:C\3\2\2\2;<\7\21\2\2<=\5\b\5\2=>\7\22\2\2>?\5\4\3\2"+ + "?@\7\23\2\2@A\5\4\3\2AC\3\2\2\2B\65\3\2\2\2B;\3\2\2\2C\7\3\2\2\2DE\b\5"+ + "\1\2EF\5\n\6\2FG\5\16\b\2GH\5\n\6\2H\\\3\2\2\2IJ\5&\24\2JK\7\37\2\2KL"+ + "\5&\24\2L\\\3\2\2\2MN\5&\24\2NO\7 \2\2OP\5&\24\2P\\\3\2\2\2QR\7\6\2\2"+ + "RS\7\35\2\2ST\5\b\5\2TU\7\36\2\2U\\\3\2\2\2VW\7\35\2\2WX\5\b\5\2XY\7\36"+ + "\2\2Y\\\3\2\2\2Z\\\5&\24\2[D\3\2\2\2[I\3\2\2\2[M\3\2\2\2[Q\3\2\2\2[V\3"+ + "\2\2\2[Z\3\2\2\2\\e\3\2\2\2]^\f\7\2\2^_\7\4\2\2_d\5\b\5\b`a\f\6\2\2ab"+ + "\7\5\2\2bd\5\b\5\7c]\3\2\2\2c`\3\2\2\2dg\3\2\2\2ec\3\2\2\2ef\3\2\2\2f"+ + "\t\3\2\2\2ge\3\2\2\2hi\5&\24\2i\13\3\2\2\2jk\5&\24\2k\r\3\2\2\2lm\t\2"+ + "\2\2m\17\3\2\2\2no\t\3\2\2o\21\3\2\2\2pq\t\4\2\2q\23\3\2\2\2rs\7\35\2"+ + "\2st\5\26\f\2tu\7\36\2\2uy\3\2\2\2vw\7\35\2\2wy\7\36\2\2xr\3\2\2\2xv\3"+ + "\2\2\2y\25\3\2\2\2z{\b\f\1\2{|\5&\24\2|\u0082\3\2\2\2}~\f\3\2\2~\177\7"+ + "\3\2\2\177\u0081\5&\24\2\u0080}\3\2\2\2\u0081\u0084\3\2\2\2\u0082\u0080"+ + "\3\2\2\2\u0082\u0083\3\2\2\2\u0083\27\3\2\2\2\u0084\u0082\3\2\2\2\u0085"+ + "\u0086\7\33\2\2\u0086\u0087\5\26\f\2\u0087\u0088\7\34\2\2\u0088\u008c"+ + "\3\2\2\2\u0089\u008a\7\33\2\2\u008a\u008c\7\34\2\2\u008b\u0085\3\2\2\2"+ + "\u008b\u0089\3\2\2\2\u008c\31\3\2\2\2\u008d\u008e\b\16\1\2\u008e\u008f"+ + "\5&\24\2\u008f\u0090\7\20\2\2\u0090\u0091\5\4\3\2\u0091\u009a\3\2\2\2"+ + "\u0092\u0093\f\3\2\2\u0093\u0094\7\3\2\2\u0094\u0095\5&\24\2\u0095\u0096"+ + "\7\20\2\2\u0096\u0097\5\4\3\2\u0097\u0099\3\2\2\2\u0098\u0092\3\2\2\2"+ + "\u0099\u009c\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2\2\2\u009b\33"+ + "\3\2\2\2\u009c\u009a\3\2\2\2\u009d\u009e\7\31\2\2\u009e\u009f\5\32\16"+ + "\2\u009f\u00a0\7\32\2\2\u00a0\u00a4\3\2\2\2\u00a1\u00a2\7\31\2\2\u00a2"+ + "\u00a4\7\32\2\2\u00a3\u009d\3\2\2\2\u00a3\u00a1\3\2\2\2\u00a4\35\3\2\2"+ + "\2\u00a5\u00a6\b\20\1\2\u00a6\u00a7\5 \21\2\u00a7\u00b0\3\2\2\2\u00a8"+ + "\u00a9\f\4\2\2\u00a9\u00aa\7\26\2\2\u00aa\u00af\5 \21\2\u00ab\u00ac\f"+ + "\3\2\2\u00ac\u00ad\7\25\2\2\u00ad\u00af\5 \21\2\u00ae\u00a8\3\2\2\2\u00ae"+ + "\u00ab\3\2\2\2\u00af\u00b2\3\2\2\2\u00b0\u00ae\3\2\2\2\u00b0\u00b1\3\2"+ + "\2\2\u00b1\37\3\2\2\2\u00b2\u00b0\3\2\2\2\u00b3\u00b4\b\21\1\2\u00b4\u00b5"+ + "\5$\23\2\u00b5\u00be\3\2\2\2\u00b6\u00b7\f\4\2\2\u00b7\u00b8\7\30\2\2"+ + "\u00b8\u00bd\5 \21\5\u00b9\u00ba\f\3\2\2\u00ba\u00bb\7\27\2\2\u00bb\u00bd"+ + "\5 \21\4\u00bc\u00b6\3\2\2\2\u00bc\u00b9\3\2\2\2\u00bd\u00c0\3\2\2\2\u00be"+ + "\u00bc\3\2\2\2\u00be\u00bf\3\2\2\2\u00bf!\3\2\2\2\u00c0\u00be\3\2\2\2"+ + "\u00c1\u00c2\7$\2\2\u00c2\u00c3\5\24\13\2\u00c3#\3\2\2\2\u00c4\u00cd\5"+ + "\"\22\2\u00c5\u00cd\7#\2\2\u00c6\u00cd\7\"\2\2\u00c7\u00cd\7$\2\2\u00c8"+ + "\u00c9\7\35\2\2\u00c9\u00ca\5\36\20\2\u00ca\u00cb\7\36\2\2\u00cb\u00cd"+ + "\3\2\2\2\u00cc\u00c4\3\2\2\2\u00cc\u00c5\3\2\2\2\u00cc\u00c6\3\2\2\2\u00cc"+ + "\u00c7\3\2\2\2\u00cc\u00c8\3\2\2\2\u00cd%\3\2\2\2\u00ce\u00d9\t\5\2\2"+ + "\u00cf\u00d9\5\36\20\2\u00d0\u00d9\7%\2\2\u00d1\u00d9\5\30\r\2\u00d2\u00d9"+ + "\5\34\17\2\u00d3\u00d9\7\24\2\2\u00d4\u00d5\7!\2\2\u00d5\u00d6\7\35\2"+ + "\2\u00d6\u00d7\7$\2\2\u00d7\u00d9\7\36\2\2\u00d8\u00ce\3\2\2\2\u00d8\u00cf"+ + "\3\2\2\2\u00d8\u00d0\3\2\2\2\u00d8\u00d1\3\2\2\2\u00d8\u00d2\3\2\2\2\u00d8"+ + "\u00d3\3\2\2\2\u00d8\u00d4\3\2\2\2\u00d9\'\3\2\2\2\22\63B[cex\u0082\u008b"+ + "\u009a\u00a3\u00ae\u00b0\u00bc\u00be\u00cc\u00d8"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java index fe87c55bff..ba03b7063d 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.Context; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -42,6 +43,7 @@ public Map map( Map input , List outputField , Map fieldMappingConfig , Map sensorConfig + , Context context ) { return ImmutableMap.of(outputField.get(0), Joiner.on(fieldMappingConfig.get("delim").toString()).join(input.entrySet())); @@ -132,6 +134,7 @@ public void testComplexMapping() throws IOException { ) ) , c.getParserConfig() + , Context.EMPTY_CONTEXT() ) ); } @@ -142,7 +145,7 @@ public void testSimpleMapping() throws IOException { Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") - ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)), c.getParserConfig()) + ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)), c.getParserConfig(), Context.EMPTY_CONTEXT()) ); } } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RemoveTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RemoveTransformationTest.java index fc43406b1b..fcd006dac9 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RemoveTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RemoveTransformationTest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.Context; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -50,7 +51,7 @@ public void testUnconditionalRemove() throws Exception{ JSONObject input = new JSONObject(new HashMap() {{ put("field1", "foo"); }}); - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); } @@ -77,7 +78,7 @@ public void testConditionalRemove() throws Exception { JSONObject input = new JSONObject(new HashMap() {{ put("field1", "foo"); }}); - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); //no removal happened because field2 does not exist Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); @@ -87,7 +88,7 @@ public void testConditionalRemove() throws Exception { put("field1", "foo"); put("field2", "bar"); }}); - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); //no removal happened because field2 != bar Assert.assertTrue(input.containsKey("field1")); Assert.assertTrue(input.containsKey("field2")); @@ -98,7 +99,7 @@ public void testConditionalRemove() throws Exception { put("field2", "foo"); }}); //removal of field1 happens because field2 exists and is 'bar' - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); Assert.assertTrue(input.containsKey("field2")); } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index 7cb187973a..f01e042c6f 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.Context; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -57,7 +58,7 @@ public void testStellar() throws Exception { JSONObject input = new JSONObject(new HashMap() {{ put("timestamp", "2016-01-05 17:02:30"); }}); - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); @@ -75,7 +76,7 @@ public void testStellar_negative() throws Exception { //no input fields => no transformation JSONObject input = new JSONObject(new HashMap() {{ }}); - handler.transformAndUpdate(input, new HashMap<>()); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("utc_timestamp")); Assert.assertTrue(input.isEmpty()); } @@ -126,7 +127,7 @@ public void testStellar_multi() throws Exception { //looking up the data center in portland, which doesn't exist in the map, so we default to UTC put("dc", "portland"); }}); - handler.transformAndUpdate(input, c.getParserConfig()); + handler.transformAndUpdate(input, c.getParserConfig(), Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertEquals("caseystella.com", input.get("url_host")); @@ -141,7 +142,7 @@ public void testStellar_multi() throws Exception { put("url", "https://caseystella.com/blog"); put("dc", "london"); }}); - handler.transformAndUpdate(input, c.getParserConfig()); + handler.transformAndUpdate(input, c.getParserConfig(), Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertEquals("caseystella.com", input.get("url_host")); @@ -155,7 +156,7 @@ public void testStellar_multi() throws Exception { put("timestamp", "2016-01-05 17:02:30"); put("url", "https://caseystella.com/blog"); }}); - handler.transformAndUpdate(input, c.getParserConfig()); + handler.transformAndUpdate(input, c.getParserConfig(), Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertEquals("caseystella.com", input.get("url_host")); diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/BaseValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/BaseValidationTest.java index ab78c3cbac..22cb47a2fd 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/BaseValidationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/BaseValidationTest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.FieldValidator; +import org.apache.metron.common.dsl.Context; import org.json.simple.JSONObject; import java.io.IOException; @@ -42,7 +43,9 @@ public boolean execute(String config, Map input) throws IOExcept FieldValidator validator = getValidator(configurations); return validator.isValid(new JSONObject(input) - ,configurations.getGlobalConfig() + ,configurations.getGlobalConfig() + , Context.EMPTY_CONTEXT() + ); } } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java index 70d540f0db..abcf204497 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java @@ -20,9 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.common.dsl.*; import org.junit.Assert; import org.junit.Test; @@ -31,8 +29,75 @@ public class StellarTest { + @Test + public void testFunctionEmptyArgs() { + { + String query = "STARTS_WITH(casey, 'case') or MAP_EXISTS()"; + Assert.assertTrue((Boolean)run(query, ImmutableMap.of("casey", "casey"))); + } + { + String query = "true or MAP_EXISTS()"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + { + String query = "MAP_EXISTS() or true"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + } + @Test + public void testNull() { + { + String query = "if 1 < 2 then NULL else true"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? NULL : true"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "null == null ? true : false"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + } + + @Test + public void testMapConstant() { + { + String query = "MAP_GET('bar', { 'foo' : 1, 'bar' : 'bar'})"; + Assert.assertEquals("bar", run(query, new HashMap<>())); + } + { + String query = "MAP_GET('blah', { 'blah' : 1 < 2 })"; + Assert.assertEquals(true, run(query, new HashMap<>())); + } + { + String query = "MAP_GET('blah', { 'blah' : not(STARTS_WITH(casey, 'case')) })"; + Assert.assertEquals(false, run(query, ImmutableMap.of("casey", "casey"))); + } + { + String query = "MAP_GET('blah', { 'blah' : one })"; + Assert.assertEquals(1, run(query, ImmutableMap.of("one", 1))); + } + { + String query = "MAP_GET('blah', { 'blah' : null })"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "MAP_GET('BLAH', { TO_UPPER('blah') : null })"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "MAP_GET('BLAH', { TO_UPPER('blah') : 1 < 2 })"; + Assert.assertEquals(true, run(query, new HashMap<>())); + } + } + @Test public void testIfThenElse() { + { + String query = "if STARTS_WITH(casey, 'case') then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("casey", "casey"))); + } { String query = "if 1 < 2 then 'one' else 'two'"; Assert.assertEquals("one", run(query, new HashMap<>())); @@ -41,10 +106,7 @@ public void testIfThenElse() { String query = "if 1 + 1 < 2 then 'one' else 'two'"; Assert.assertEquals("two", run(query, new HashMap<>())); } - { - String query = "1 < 2 ? 'one' : 'two'"; - Assert.assertEquals("one", run(query, new HashMap<>())); - } + { String query = "if not(1 < 2) then 'one' else 'two'"; Assert.assertEquals("two", run(query, new HashMap<>())); @@ -61,6 +123,22 @@ public void testIfThenElse() { String query = "if one == very_nearly_one then 'one' else 'two'"; Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.000001))); } + { + String query = "1 < 2 ? 'one' : 'two'"; + Assert.assertEquals("one", run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? TO_UPPER('one') : 'two'"; + Assert.assertEquals("ONE", run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? one : 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", "one"))); + } + { + String query = "1 < 2 ? one*3 : 'two'"; + Assert.assertTrue(Math.abs(3.0 - (double)run(query, ImmutableMap.of("one", 1))) < 1e-6); + } } @Test @@ -232,12 +310,17 @@ public void testGet() { Assert.assertEquals("google", run("GET(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'), 1)", variables)); } - private static Object run(String rule, Map variables) { + + + public static Object run(String rule, Map variables) { + return run(rule, variables, Context.EMPTY_CONTEXT()); + } + public static Object run(String rule, Map variables, Context context) { StellarProcessor processor = new StellarProcessor(); Assert.assertTrue(rule + " not valid.", processor.validate(rule)); - return processor.parse(rule, x -> variables.get(x)); + return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context); } - + @Test public void testValidation() throws Exception { StellarPredicateProcessor processor = new StellarPredicateProcessor(); @@ -258,13 +341,21 @@ public void testValidation() throws Exception { } public static boolean runPredicate(String rule, Map resolver) { - return runPredicate(rule, new MapVariableResolver(resolver)); + return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); + } + + public static boolean runPredicate(String rule, Map resolver, Context context) { + return runPredicate(rule, new MapVariableResolver(resolver), context); } public static boolean runPredicate(String rule, VariableResolver resolver) { + return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); + } + + public static boolean runPredicate(String rule, VariableResolver resolver, Context context) { StellarPredicateProcessor processor = new StellarPredicateProcessor(); Assert.assertTrue(rule + " not valid.", processor.validate(rule)); - return processor.parse(rule, resolver); + return processor.parse(rule, resolver, StellarFunctions.FUNCTION_RESOLVER(), context); } @Test diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/MockDGAModel.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/MockDGAModel.java new file mode 100644 index 0000000000..0373c27809 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/MockDGAModel.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.stellar.maas; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.metron.common.utils.JSONUtils; + +import javax.ws.rs.*; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.ext.RuntimeDelegate; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.*; + +@Path("/") +public class MockDGAModel { + private static HttpServer server; + private Map isMalicious = ImmutableMap.of( "badguy.com", true + ); + + @GET + @Path("/apply") + @Produces("application/json") + public Response apply(@QueryParam("host") String host ) throws JsonProcessingException { + Boolean b = isMalicious.get(host); + boolean isMalicious = b != null && b; + Map ret = new HashMap(); + ret.put("is_malicious", isMalicious ); + String resp = JSONUtils.INSTANCE.toJSON(ret, true); + return Response.ok(resp, MediaType.APPLICATION_JSON_TYPE).build(); + } + + @ApplicationPath("rs") + public static class ApplicationConfig extends Application { + private final Set> classes; + public ApplicationConfig() { + HashSet> c = new HashSet<>(); + c.add(MockDGAModel.class); + classes = Collections.unmodifiableSet(c); + } + @Override + public Set> getClasses() { + return classes; + } + } + + public static void start(int port) throws IOException { + // Create an HTTP server listening at port + URI uri = UriBuilder.fromUri("http://localhost/").port(port).build(); + server = HttpServer.create(new InetSocketAddress(uri.getPort()), 0); + HttpHandler handler = RuntimeDelegate.getInstance().createEndpoint(new ApplicationConfig(), HttpHandler.class); + server.createContext(uri.getPath(), handler); + server.start(); + } + + public static void shutdown() { + if(server != null) { + server.stop(0); + } + } +} diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java new file mode 100644 index 0000000000..82114f7a55 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.stellar.maas; + +import com.google.common.collect.ImmutableMap; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingServer; +import org.apache.curator.utils.CloseableUtils; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceInstanceBuilder; +import org.apache.curator.x.discovery.ServiceType; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.common.stellar.StellarTest; +import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.RESTUtil; +import org.junit.*; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class StellarMaaSIntegrationTest { + private static Context context; + private static TestingServer testZkServer; + private static String zookeeperUrl; + private static CuratorFramework client; + private static ServiceDiscoverer discoverer; + private static URL endpointUrl; + + @BeforeClass + public static void setup() throws Exception { + MockDGAModel.start(8282); + testZkServer = new TestingServer(true); + zookeeperUrl = testZkServer.getConnectString(); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); + client.start(); + context = new Context.Builder() + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) + .build(); + StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(context); + discoverer = (ServiceDiscoverer) context.getCapability(Context.Capabilities.SERVICE_DISCOVERER).get(); + endpointUrl = new URL("http://localhost:8282"); + ModelEndpoint endpoint = new ModelEndpoint(); + { + endpoint.setName("dga"); + endpoint.setContainerId("0"); + Endpoint ep = new Endpoint(); + ep.setUrl(endpointUrl.toString()); + endpoint.setEndpoint(ep); + endpoint.setVersion("1.0"); + } + ; + + ServiceInstanceBuilder builder = ServiceInstance.builder() + .address(endpointUrl.getHost()) + .id("0") + .name("dga") + .port(endpointUrl.getPort()) + .registrationTimeUTC(System.currentTimeMillis()) + .serviceType(ServiceType.STATIC) + .payload(endpoint); + final ServiceInstance instance = builder.build(); + discoverer.getServiceDiscovery().registerService(instance); + //wait til the endpoint is installed... + for(int i = 0;i < 10;++i) { + try { + Object o = discoverer.getEndpoint("dga"); + if(o != null) { + break; + } + } + catch(Exception e) { + + } + Thread.sleep(1000); + } + } + + @Test + public void testGetEndpointWithoutVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga')"; + Object result = StellarTest.run(stellar, new HashMap<>(), context); + Assert.assertTrue(result instanceof Map); + Map resMap = (Map)result; + Assert.assertEquals(resMap.get("url"), "http://localhost:8282"); + Assert.assertEquals(resMap.get("name"), "dga"); + Assert.assertEquals(resMap.get("version"), "1.0"); + Assert.assertEquals(resMap.get("endpoint:apply"), "apply"); + + } + + @Test + public void testGetEndpointWithVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga', '1.0')"; + Object result = StellarTest.run(stellar, new HashMap<>(), context); + Assert.assertTrue(result instanceof Map); + Map resMap = (Map)result; + Assert.assertEquals(resMap.get("url"), "http://localhost:8282"); + Assert.assertEquals(resMap.get("name"), "dga"); + Assert.assertEquals(resMap.get("version"), "1.0"); + Assert.assertEquals(resMap.get("endpoint:apply"), "apply"); + } + + @Test + public void testGetEndpointWithWrongVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga', '2.0')"; + Object result = StellarTest.run(stellar, new HashMap<>(), context); + Assert.assertNull(result); + } + + @Test + public void testModelApply() throws Exception { + { + String stellar = "MAP_GET('is_malicious', MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))"; + Object result = StellarTest.run(stellar, ImmutableMap.of("host", "badguy.com"), context); + Assert.assertTrue((Boolean) result); + } + { + String stellar = "MAP_GET('is_malicious', MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))"; + Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertFalse((Boolean) result); + } + { + String stellar = "MAP_GET('is_malicious', MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), 'apply', {'host': host}))"; + Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertFalse((Boolean) result); + } + + } + + @Test + public void testModelApplyNegative() { + { + String stellar = "MAP_GET('is_malicious', MODEL_APPLY(MAAS_GET_ENDPOINT('dga', '2.0'), {'host': host}))"; + Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertNull( result); + } + } + + @AfterClass + public static void teardown() { + MockDGAModel.shutdown(); + if(discoverer != null) { + CloseableUtils.closeQuietly(discoverer); + } + if(client != null) { + CloseableUtils.closeQuietly(client); + } + if(testZkServer != null) { + CloseableUtils.closeQuietly(testZkServer); + } + } +} diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 55e9d9234f..2ea05e7187 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -17,10 +17,15 @@ */ package org.apache.metron.enrichment.bolt; +import backtype.storm.task.TopologyContext; import com.google.common.base.Joiner; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; +import org.apache.metron.common.dsl.FunctionResolver; +import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.common.dsl.functions.ConversionFunctions; +import org.apache.metron.common.utils.ConversionUtils; import org.apache.metron.common.utils.MessageUtils; import org.apache.metron.threatintel.triage.ThreatTriageProcessor; import org.json.simple.JSONObject; @@ -34,11 +39,18 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt { protected static final Logger LOG = LoggerFactory .getLogger(ThreatIntelJoinBolt.class); - + FunctionResolver functionResolver; public ThreatIntelJoinBolt(String zookeeperUrl) { super(zookeeperUrl); } + @Override + public void prepare(Map map, TopologyContext topologyContext) { + super.prepare(map, topologyContext); + functionResolver = StellarFunctions.FUNCTION_RESOLVER(); + functionResolver.initializeFunctions(context); + } + @Override public Map> getFieldMap(String sourceType) { SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType); @@ -63,6 +75,13 @@ public JSONObject joinMessages(Map streamMessageMap) { } } } + else { + Object isAlertObj = ret.get("is_alert"); + isAlert = ConversionUtils.convert(isAlertObj, Boolean.class); + if(!isAlert) { + ret.remove("is_alert"); + } + } if(isAlert) { ret.put("is_alert" , "true"); String sourceType = MessageUtils.getSensorType(ret); @@ -86,7 +105,7 @@ public JSONObject joinMessages(Map streamMessageMap) { LOG.debug(sourceType + ": Empty rules!"); } - ThreatTriageProcessor threatTriageProcessor = new ThreatTriageProcessor(config); + ThreatTriageProcessor threatTriageProcessor = new ThreatTriageProcessor(config, functionResolver, context); Double triageLevel = threatTriageProcessor.apply(ret); if(LOG.isDebugEnabled()) { String rules = Joiner.on('\n').join(triageConfig.getRiskLevelRules().entrySet()); diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java index f3e645cf66..137ecbbe68 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java @@ -22,9 +22,8 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; -import org.apache.metron.common.dsl.MapVariableResolver; +import org.apache.metron.common.dsl.*; import org.apache.metron.common.stellar.StellarPredicateProcessor; -import org.apache.metron.common.dsl.VariableResolver; import javax.annotation.Nullable; import java.util.ArrayList; @@ -35,10 +34,18 @@ public class ThreatTriageProcessor implements Function { private SensorEnrichmentConfig sensorConfig; private ThreatIntelConfig threatIntelConfig; private ThreatTriageConfig threatTriageConfig; - public ThreatTriageProcessor(SensorEnrichmentConfig config) { + private Context context; + private FunctionResolver functionResolver; + public ThreatTriageProcessor( SensorEnrichmentConfig config + , FunctionResolver functionResolver + , Context context + ) + { this.threatIntelConfig = config.getThreatIntel(); this.sensorConfig = config; this.threatTriageConfig = config.getThreatIntel().getTriageConfig(); + this.functionResolver = functionResolver; + this.context = context; } @Nullable @@ -48,7 +55,7 @@ public Double apply(@Nullable Map input) { StellarPredicateProcessor predicateProcessor = new StellarPredicateProcessor(); VariableResolver resolver = new MapVariableResolver(input, sensorConfig.getConfiguration(), threatIntelConfig.getConfig()); for(Map.Entry kv : threatTriageConfig.getRiskLevelRules().entrySet()) { - if(predicateProcessor.parse(kv.getKey(), resolver)) { + if(predicateProcessor.parse(kv.getKey(), resolver, functionResolver, context)) { scores.add(kv.getValue()); } } diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java index 8ae74b341c..37292e2d40 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java @@ -21,6 +21,8 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; @@ -53,7 +55,7 @@ public class ThreatTriageTest { private static ThreatTriageProcessor getProcessor(String config) throws IOException { SensorEnrichmentConfig c = JSONUtils.INSTANCE.load(config, SensorEnrichmentConfig.class); - return new ThreatTriageProcessor(c); + return new ThreatTriageProcessor(c, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); } @Test @@ -61,7 +63,7 @@ public void smokeTest() throws Exception { ThreatTriageProcessor threatTriageProcessor = getProcessor(smokeTestProcessorConfig); Assert.assertEquals("Expected a score of 0" , 0d - ,new ThreatTriageProcessor(new SensorEnrichmentConfig()).apply(new HashMap() {{ + ,new ThreatTriageProcessor(new SensorEnrichmentConfig(), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()).apply(new HashMap() {{ put("user.type", "admin"); put("asset.type", "web"); }} diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 6c40768108..746ec18e0f 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -27,6 +27,8 @@ import org.apache.metron.common.bolt.ConfiguredParserBolt; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.FunctionResolver; +import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.parsers.filters.Filters; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.parsers.filters.GenericMessageFilter; @@ -89,6 +91,7 @@ else if(filter == null) { throw new IllegalStateException("Unable to retrieve a parser config for " + getSensorType()); } parser.configure(config.getParserConfig()); + StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(super.context); } @@ -106,11 +109,11 @@ public void execute(Tuple tuple) { List fieldValidations = getConfigurations().getFieldValidations(); Optional> messages = parser.parseOptional(originalMessage); for (JSONObject message : messages.orElse(Collections.emptyList())) { - if (parser.validate(message) && filter != null && filter.emitTuple(message)) { + if (parser.validate(message) && filter != null && filter.emitTuple(message, super.context)) { message.put(Constants.SENSOR_TYPE, getSensorType()); for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { - handler.transformAndUpdate(message, sensorParserConfig.getParserConfig()); + handler.transformAndUpdate(message, sensorParserConfig.getParserConfig(), super.context); } } numWritten++; @@ -143,7 +146,7 @@ public void execute(Tuple tuple) { private boolean isGloballyValid(JSONObject input, List validators) { for(FieldValidator validator : validators) { - if(!validator.isValid(input, getConfigurations().getGlobalConfig())) { + if(!validator.isValid(input, getConfigurations().getGlobalConfig(), super.context)) { return false; } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java index 2145dfeb5a..87129fcf13 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.filters; import org.apache.commons.configuration.Configuration; +import org.apache.metron.common.dsl.Context; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; @@ -66,7 +67,7 @@ else if(protocolsObj instanceof List) { * @return False if message if filtered and True if message is not filtered */ - public boolean emitTuple(JSONObject message) { + public boolean emitTuple(JSONObject message, Context context) { String protocol = (String) message.get(_key); return _known_protocols.contains(protocol); } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/GenericMessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/GenericMessageFilter.java index e4524ae2cf..3104180d7e 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/GenericMessageFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/GenericMessageFilter.java @@ -17,6 +17,7 @@ */ package org.apache.metron.parsers.filters; +import org.apache.metron.common.dsl.Context; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; @@ -30,7 +31,7 @@ public class GenericMessageFilter implements MessageFilter{ public GenericMessageFilter() { } - public boolean emitTuple(JSONObject message) { + public boolean emitTuple(JSONObject message, Context context) { return true; } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java index 4ea574887c..0ec22e2f6b 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java @@ -18,9 +18,8 @@ package org.apache.metron.parsers.filters; -import org.apache.metron.common.dsl.MapVariableResolver; +import org.apache.metron.common.dsl.*; import org.apache.metron.common.stellar.StellarPredicateProcessor; -import org.apache.metron.common.dsl.VariableResolver; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; @@ -30,6 +29,7 @@ public class QueryFilter implements MessageFilter { public static final String QUERY_STRING_CONF = "filter.query"; private StellarPredicateProcessor processor = new StellarPredicateProcessor(); private String query; + private FunctionResolver functionResolver = StellarFunctions.FUNCTION_RESOLVER(); public QueryFilter() { @@ -47,8 +47,8 @@ public void configure(Map config) { } @Override - public boolean emitTuple(JSONObject message) { + public boolean emitTuple(JSONObject message, Context context) { VariableResolver resolver = new MapVariableResolver(message); - return processor.parse(query, resolver); + return processor.parse(query, resolver, functionResolver, context); } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java index 10a899b88f..abcb57ae9c 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java @@ -17,10 +17,12 @@ */ package org.apache.metron.parsers.interfaces; +import org.apache.metron.common.dsl.Context; + import java.io.Serializable; import java.util.Map; public interface MessageFilter extends Configurable{ - boolean emitTuple(T message); + boolean emitTuple(T message, Context context); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java index 9df0101d64..dde9f7ddee 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.logging.log4j.core.filter.AbstractFilter; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.Context; import org.apache.metron.parsers.filters.AbstractMessageFilter; import org.apache.metron.parsers.filters.Filters; import org.apache.metron.parsers.filters.GenericMessageFilter; @@ -38,8 +39,8 @@ public class FiltersTest { @Test public void testDefault() { - Assert.assertTrue(Filters.get("DEFAULT", null).emitTuple(null)); - Assert.assertTrue(Filters.get(GenericMessageFilter.class.getName(), null).emitTuple(null)); + Assert.assertTrue(Filters.get("DEFAULT", null).emitTuple(null, null)); + Assert.assertTrue(Filters.get(GenericMessageFilter.class.getName(), null).emitTuple(null, null)); } @Test @@ -49,8 +50,8 @@ public void testSingleQueryFilter() throws Exception { put("filter.query", "exists(foo)"); }}; MessageFilter filter = Filters.get("QUERY", config); - Assert.assertTrue(filter.emitTuple(new JSONObject(ImmutableMap.of("foo", 1)))); - Assert.assertFalse(filter.emitTuple(new JSONObject(ImmutableMap.of("bar", 1)))); + Assert.assertTrue(filter.emitTuple(new JSONObject(ImmutableMap.of("foo", 1)), Context.EMPTY_CONTEXT())); + Assert.assertFalse(filter.emitTuple(new JSONObject(ImmutableMap.of("bar", 1)), Context.EMPTY_CONTEXT())); } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 0b07bc3edf..3cb7ae6fd8 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -26,6 +26,7 @@ import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.interfaces.BulkMessageWriter; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.hbase.util.Bytes; @@ -166,8 +167,8 @@ public Map getParserConfig() { verify(outputCollector, times(1)).ack(tuple); when(parser.validate(eq(messages.get(0)))).thenReturn(true); when(parser.validate(eq(messages.get(1)))).thenReturn(true); - when(filter.emitTuple(messages.get(0))).thenReturn(false); - when(filter.emitTuple(messages.get(1))).thenReturn(true); + when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false); + when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(tuple); verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2)); @@ -205,7 +206,7 @@ public Map getParserConfig() { verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); - when(filter.emitTuple(any())).thenReturn(true); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); verify(outputCollector, times(1)).ack(t1); @@ -276,7 +277,7 @@ public Map getParserConfig() { verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); - when(filter.emitTuple(any())).thenReturn(true); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); verify(outputCollector, times(1)).ack(t1); @@ -311,7 +312,7 @@ public Map getParserConfig() { verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); - when(filter.emitTuple(any())).thenReturn(true); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); writeNonBatch(outputCollector, parserBolt, t1); writeNonBatch(outputCollector, parserBolt, t2); @@ -357,7 +358,7 @@ public Map getParserConfig() { doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any()); when(parser.validate(any())).thenReturn(true); when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject())); - when(filter.emitTuple(any())).thenReturn(true); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); parserBolt.execute(t2); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index afdcf0424a..b9377f3ff2 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.integration.components.ConfigUploadComponent; @@ -41,7 +42,7 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest { public static class MockValidator implements FieldValidation{ @Override - public boolean isValid(Map input, Map validationConfig, Map globalConfig) { + public boolean isValid(Map input, Map validationConfig, Map globalConfig, Context context) { if(input.get("action").equals("invalid")) { return false; } diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java index e2a9e11ac8..28a8b93985 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java @@ -20,6 +20,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.metron.common.Constants; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.stellar.StellarPredicateProcessor; import org.apache.metron.common.dsl.VariableResolver; import org.apache.metron.pcap.PacketInfo; @@ -69,7 +71,7 @@ public void configure(Iterable> config) { public boolean test(PacketInfo input) { EnumMap fields = packetToFields(input); VariableResolver resolver = new PcapFieldResolver(fields); - return predicateProcessor.parse(queryString, resolver); + return predicateProcessor.parse(queryString, resolver, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); } protected EnumMap packetToFields(PacketInfo pi) { From 3064d7472f7a05982bc46ef6eafa93e92d483f7c Mon Sep 17 00:00:00 2001 From: Casey Stella Date: Thu, 18 Aug 2016 16:44:09 -0400 Subject: [PATCH 2/3] Update README.md --- metron-platform/metron-common/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md index 3af3be89ba..2137469a43 100644 --- a/metron-platform/metron-common/README.md +++ b/metron-platform/metron-common/README.md @@ -47,6 +47,8 @@ The query language supports the following: * `URL_TO_PORT(url)` : Returns the port from a URL * `URL_TO_PATH(url)` : Returns the path from a URL * `TO_EPOCH_TIMESTAMP(dateTime, format, timezone)` : Returns the epoch timestamp of the `dateTime` given the `format`. If the format does not have a timestamp and you wish to assume a given timestamp, you may specify the `timezone` optionally. + * `MODEL_APPLY(endpoint, function?, model_args)` : Returns the output of a model deployed via model which is deployed at endpoint. `endpoint` is a map containing `name`, `version`, `url` for the REST endpoint, `function` is the endpoint path and is optional, and `model_args` is a dictionary of arguments for the model (these become request params). + * `MAAS_GET_ENDPOINT(model_name, model_version?)` : Inspects zookeeper and returns a map containing the `name`, `version` and `url` for the model referred to by `model_name` and `model_version`. If `model_version` is not specified, the most current model associated with `model_name` is returned. In the instance where more than one model is deployed, a random one is selected with uniform probability. The following is an example query (i.e. a function which returns a boolean) which would be seen possibly in threat triage: From 411d34426a911e59aa57ebe439c6635b0f00f665 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 19 Aug 2016 18:43:17 -0400 Subject: [PATCH 3/3] Reacted to PR requests for better docs and code clarity improvements. --- .../org/apache/metron/maas/config/Action.java | 3 + .../apache/metron/maas/config/Endpoint.java | 36 +++-- .../apache/metron/maas/config/MaaSConfig.java | 12 ++ .../metron/maas/config/ModelEndpoint.java | 2 +- .../metron/maas/config/ModelRequest.java | 27 ++++ .../maas/discovery/ServiceDiscoverer.java | 126 ++++++++++++++++-- .../org/apache/metron/maas/queue/Queue.java | 4 + .../metron/maas/queue/QueueHandler.java | 12 ++ .../src/test/resources/maas/dummy_rest.sh | 2 +- .../common/dsl/functions/MaaSFunctions.java | 3 +- .../enrichment/bolt/EnrichmentJoinBolt.java | 2 +- 11 files changed, 202 insertions(+), 27 deletions(-) diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Action.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Action.java index 2ddbbd93e7..07f2659018 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Action.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Action.java @@ -17,6 +17,9 @@ */ package org.apache.metron.maas.config; +/** + * The actions available to be taken by the Model as a Service ApplicationMaster + */ public enum Action { ADD, REMOVE; } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java index 0ba1a18561..b865a0f5af 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/Endpoint.java @@ -20,12 +20,20 @@ import java.util.HashMap; import java.util.Map; +/** + * An endpoint for a service being deployed. An endpoint is constituted by a URL and + * a set of functions which it exposes. + */ public class Endpoint { String url; - Map endpoints = new HashMap(){{ + Map functions = new HashMap(){{ put("apply", "apply"); }}; + /** + * Retrieve the URL associated with the endpoint + * @return a URL + */ public String getUrl() { return url; } @@ -34,12 +42,24 @@ public void setUrl(String url) { this.url = url; } - public Map getEndpoints() { - return endpoints; + /** + * Retrieve the functions (or functions) exposed. The key of the map is the logical alias + * for an endpoint function that you can use across multiple services (i.e. 'apply') + * and the value is the realized endpoint function name. + * + * For instance, if I have a service bound to localhost:8080 and exposes /foo and /bar and + * for your organization or use you've decided to call functions which are used for enrichment 'enrich' + * and functions which are used for data science as 'ds', you can alias /foo to 'enrich' + * and /bar to 'ds' by creating a map which maps the 'enrich' to 'foo' and 'ds' to 'bar'. + * + * @return + */ + public Map getFunctions() { + return functions; } - public void setEndpoints(Map endpoints) { - this.endpoints = endpoints; + public void setFunctions(Map functions) { + this.functions = functions; } @@ -47,7 +67,7 @@ public void setEndpoints(Map endpoints) { public String toString() { return "Endpoint{" + "url='" + url + '\'' + - ", endpoints=" + endpoints + + ", functions=" + functions + '}'; } @@ -59,14 +79,14 @@ public boolean equals(Object o) { Endpoint endpoint = (Endpoint) o; if (getUrl() != null ? !getUrl().equals(endpoint.getUrl()) : endpoint.getUrl() != null) return false; - return getEndpoints() != null ? getEndpoints().equals(endpoint.getEndpoints()) : endpoint.getEndpoints() == null; + return getFunctions() != null ? getFunctions().equals(endpoint.getFunctions()) : endpoint.getFunctions() == null; } @Override public int hashCode() { int result = getUrl() != null ? getUrl().hashCode() : 0; - result = 31 * result + (getEndpoints() != null ? getEndpoints().hashCode() : 0); + result = 31 * result + (getFunctions() != null ? getFunctions().hashCode() : 0); return result; } } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/MaaSConfig.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/MaaSConfig.java index 5244468e7f..b195478371 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/MaaSConfig.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/MaaSConfig.java @@ -24,6 +24,9 @@ import java.util.HashMap; import java.util.Map; +/** + * The base configuration for Model as a Service + */ public class MaaSConfig { private QueueHandler queue = QueueHandler.ZOOKEEPER; private Map queueConfig = new HashMap() {{ @@ -31,6 +34,10 @@ public class MaaSConfig { }}; private String serviceRoot = "/maas/service"; + /** + * Return the zookeeper path for the discovery service. This is defaulted to /maas/service + * @return + */ public String getServiceRoot() { return serviceRoot; } @@ -40,6 +47,11 @@ public void setServiceRoot(String serviceRoot) { } + /** + * Get the distributed queue implementation handler. By default, we use a queue in zookeeper + * as implemented by Apache Curator. + * @return + */ public QueueHandler getQueue() { return queue; } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java index 7b756b0b02..f33fbdd418 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelEndpoint.java @@ -42,7 +42,7 @@ public void setEndpoint(Endpoint endpoint) { @Override public String toString() { return name + ":" + version + " @ " + endpoint.getUrl() - + " serving:\n\t" + Joiner.on("\n\t").join(getEndpoint().getEndpoints().entrySet()); + + " serving:\n\t" + Joiner.on("\n\t").join(getEndpoint().getFunctions().entrySet()); } public void setContainerId(String containerId) { diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelRequest.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelRequest.java index 318aaa7b9f..9ba9059c34 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelRequest.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/config/ModelRequest.java @@ -17,6 +17,9 @@ */ package org.apache.metron.maas.config; +/** + * A container for a request for a model. + */ public class ModelRequest { private String name; @@ -26,6 +29,10 @@ public class ModelRequest { private Action action; private String path; + /** + * The path in the model store (right now HDFS) for the directory containing model's bits. + * @return + */ public String getPath() { return path; } @@ -34,6 +41,10 @@ public void setPath(String path) { this.path = path; } + /** + * The action to perform + * @return + */ public Action getAction() { return action; } @@ -42,6 +53,10 @@ public void setAction(Action action) { this.action = action; } + /** + * The name of the model + * @return + */ public String getName() { return name; } @@ -50,6 +65,10 @@ public void setName(String name) { this.name = name; } + /** + * The version of the model + * @return + */ public String getVersion() { return version; } @@ -58,6 +77,10 @@ public void setVersion(String version) { this.version = version; } + /** + * The number of instances of the model to start + * @return + */ public int getNumInstances() { return numInstances; } @@ -66,6 +89,10 @@ public void setNumInstances(int numInstances) { this.numInstances = numInstances; } + /** + * The amount of memory for the containers holding the model in megabytes + * @return + */ public int getMemory() { return memory; } diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java index 319b7edcd1..051e50d484 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java @@ -29,6 +29,7 @@ import org.apache.curator.x.discovery.details.JsonInstanceSerializer; import org.apache.metron.maas.config.Model; import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.zookeeper.data.Stat; import java.io.Closeable; import java.util.*; @@ -41,8 +42,12 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +/** + * The discovery class for services registered by MaaS + */ public class ServiceDiscoverer implements Closeable{ private static final Log LOG = LogFactory.getLog(ServiceDiscoverer.class); + private static final int BLACKLIST_EXPIRATION_DEFAULT = 10; private TreeCache cache; private ReadWriteLock rwLock = new ReentrantReadWriteLock(); private ServiceDiscovery serviceDiscovery; @@ -52,12 +57,31 @@ public class ServiceDiscoverer implements Closeable{ private Cache blacklist; public ServiceDiscoverer(CuratorFramework client, String root) { + this(client, root, BLACKLIST_EXPIRATION_DEFAULT); + } + + /** + * This class listens to zookeeper and updates its internal state when new model endpoints are + * added via the MaaS service. + * + * @param client The zookeeper client + * @param root The discovery root + * @param blacklistExpirationMin The amount of time (in minutes) that a blacklisted URL can be held in the blacklist before retrying. + */ + public ServiceDiscoverer(CuratorFramework client, String root, int blacklistExpirationMin) { blacklist = CacheBuilder.newBuilder() .concurrencyLevel(4) .weakKeys() - .expireAfterWrite(10, TimeUnit.MINUTES) + .expireAfterWrite(blacklistExpirationMin, TimeUnit.MINUTES) .build(); - + try { + Stat exists = client.checkExists().forPath(root); + if(exists == null) { + client.create().forPath(root); + } + } catch (Exception e) { + LOG.error("Unable to create path: " + e.getMessage(), e); + } JsonInstanceSerializer serializer = new JsonInstanceSerializer<>(ModelEndpoint.class); serviceDiscovery = ServiceDiscoveryBuilder.builder(ModelEndpoint.class) .client(client) @@ -71,6 +95,9 @@ public ServiceDiscoverer(CuratorFramework client, String root) { updateState(); } + /** + * Reset the state to empty. + */ public void resetState() { rwLock.readLock().lock(); ServiceInstance ep = null; @@ -86,6 +113,10 @@ public void resetState() { } } + /** + * Get the underlying Curator ServiceDiscovery implementation. + * @return + */ public ServiceDiscovery getServiceDiscovery() { return serviceDiscovery; } @@ -98,12 +129,20 @@ private void updateState() { for(String name : serviceDiscovery.queryForNames()) { for(ServiceInstance endpoint: serviceDiscovery.queryForInstances(name)) { ModelEndpoint ep = endpoint.getPayload(); - LOG.info("Found model endpoint " + ep); + if(LOG.isDebugEnabled()) { + LOG.debug("Found model endpoint " + ep); + } + //initialize to the existing current version, defaulting to this version String currentVersion = modelToVersion.getOrDefault(ep.getName(), ep.getVersion()); - modelToVersion.put( ep.getName() - , currentVersion.compareTo(ep.getVersion()) < 0 + //if the version for this endpont is greater than the current version, then use this one + //otherwise use the version we know about. + //essentially it's equivalent to currentVersion = max(ep.getVersion(), currentVersion) + currentVersion = currentVersion.compareTo(ep.getVersion()) < 0 ? ep.getVersion() : currentVersion + ; + modelToVersion.put( ep.getName() + , currentVersion ); containerToEndpoint.put(ep.getContainerId(), endpoint); Model model = new Model(ep.getName(), ep.getVersion()); @@ -116,10 +155,14 @@ private void updateState() { } } rwLock.writeLock().lock(); - this.modelToCurrentVersion = modelToVersion; - this.state = state; - this.containerToEndpoint = containerToEndpoint; - rwLock.writeLock().unlock(); + try { + this.modelToCurrentVersion = modelToVersion; + this.state = state; + this.containerToEndpoint = containerToEndpoint; + } + finally { + rwLock.writeLock().unlock(); + } } catch (Exception e) { LOG.error(e.getMessage(), e); } finally { @@ -128,6 +171,9 @@ private void updateState() { } + /** + * Start the discovery service + */ public void start() { try { serviceDiscovery.start(); @@ -139,6 +185,10 @@ public void start() { } } + /** + * Unregister a service based on its container ID. + * @param containerId + */ public void unregisterByContainer(String containerId) { rwLock.readLock().lock(); try { @@ -157,6 +207,12 @@ public void unregisterByContainer(String containerId) { } } + /** + * Retrieve the endpoints for a given model. A model may or may not have a version associated with it. + * If the version is missing, then all endpoints are returned associated with that model name. + * @param model + * @return + */ public List getEndpoints(Model model) { rwLock.readLock().lock(); try { @@ -167,21 +223,37 @@ public List getEndpoints(Model model) { } } + /** + * Blacklist a model endpoint + * @param endpoint + */ public void blacklist(ModelEndpoint endpoint) { blacklist(toUrl(endpoint.getEndpoint().getUrl())); } + /** + * Blacklist a model URL. + * @param url + */ public void blacklist(URL url) { rwLock.writeLock().lock(); - blacklist.put(url, true); - rwLock.writeLock().unlock(); + try { + blacklist.put(url, true); + } + finally { + rwLock.writeLock().unlock(); + } } public ModelEndpoint getEndpoint(String modelName) { String version = null; rwLock.readLock().lock(); - version = modelToCurrentVersion.get(modelName); - rwLock.readLock().unlock(); + try { + version = modelToCurrentVersion.get(modelName); + } + finally { + rwLock.readLock().unlock(); + } if(version == null) { throw new IllegalStateException("Unable to find version for " + modelName); } @@ -196,9 +268,22 @@ private static URL toUrl(String url) { } } + /** + * Retrieve an endpoint based on name and version of a model. + * This will retrieve one endpoint at random for a given model. + * @param modelName + * @param modelVersion can be null + * @return + */ public ModelEndpoint getEndpoint(String modelName, String modelVersion) { return getEndpoint(new Model(modelName, modelVersion)); } + + /** + * Retrieve an endpoint at random of a given model + * @param model + * @return + */ public ModelEndpoint getEndpoint(Model model) { rwLock.readLock().lock(); try { @@ -217,7 +302,11 @@ public ModelEndpoint getEndpoint(Model model) { } } catch(IllegalStateException ise) { - + /* + If an exception happens on an attempt then we move on. + Frankly this is an excess of caution since we parse the + URLs in the Runner before they go into zookeeper, so they are valid. + */ } } } @@ -228,6 +317,12 @@ public ModelEndpoint getEndpoint(Model model) { } } + /** + * List all endpoints for a given model. + * + * @param model + * @return + */ public Map> listEndpoints(Model model) { Map> ret = new HashMap<>(); rwLock.readLock().lock(); @@ -245,6 +340,9 @@ public Map> listEndpoints(Model model) { } } + /** + * Close the discoverer service + */ public void close() { if(cache != null) { CloseableUtils.closeQuietly(cache); diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/Queue.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/Queue.java index 0fefb3ef8e..f02319fca0 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/Queue.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/Queue.java @@ -21,6 +21,10 @@ import java.util.Map; +/** + * A basic interface for a queue of requests + * @param + */ public interface Queue { T dequeue(); void enqueue(T request); diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/QueueHandler.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/QueueHandler.java index dc472feb04..133363562c 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/QueueHandler.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/queue/QueueHandler.java @@ -20,7 +20,13 @@ import java.util.Map; import java.util.function.Function; +/** + * The set of queue handlers implemented + */ public enum QueueHandler { + /** + * A distributed queue using zookeeper. + */ ZOOKEEPER(config -> { Queue ret = new ZKQueue(); ret.configure(config); @@ -30,6 +36,12 @@ public enum QueueHandler { QueueHandler(Function, Queue> creator) { this.queueCreator = creator; } + + /** + * Create a queue handler of the specific type + * @param config + * @return + */ public Queue create(Map config) { return queueCreator.apply(config); } diff --git a/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh b/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh index 791c303a80..51c6e96fb1 100644 --- a/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh +++ b/metron-analytics/metron-maas-service/src/test/resources/maas/dummy_rest.sh @@ -3,7 +3,7 @@ rm -f out mkfifo out trap "rm -f out" EXIT -echo "{ \"url\" : \"http://localhost:1500\", \"endpoints\" : { \"apply\" : \"echo\" } }" > endpoint.dat +echo "{ \"url\" : \"http://localhost:1500\", \"functions\" : { \"apply\" : \"echo\" } }" > endpoint.dat while true do cat out | nc -l 0.0.0.0 1500 > >( # parse the netcat output, to build the answer redirected to the pipe "out". diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java index 9d14a26d6f..6fee251cfc 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -21,7 +21,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.security.authorize.Service; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.ParseException; import org.apache.metron.common.dsl.StellarFunction; @@ -230,7 +229,7 @@ public static Map endpointToMap(String name, String version, End ret.put("url", ep.getUrl()); ret.put("name", name); ret.put("version", version); - for(Map.Entry kv : ep.getEndpoints().entrySet()) { + for(Map.Entry kv : ep.getFunctions().entrySet()) { ret.put("endpoint:" + kv.getKey(), kv.getValue()); } return ret; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java index 8fa2b2ef9e..6340c956ce 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java @@ -73,7 +73,7 @@ public JSONObject joinMessages(Map streamMessageMap) { List emptyKeys = new ArrayList<>(); for(Object key : message.keySet()) { Object value = message.get(key); - if(value.toString().length() == 0) { + if(value == null || value.toString().length() == 0) { emptyKeys.add(key); } }