From 2ef7b8ff345fcb4afd2170f66e8598eef466927b Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 21 May 2018 17:23:44 -0400 Subject: [PATCH 1/4] Added a _ special variable for Stellar. --- .../enrichment/handler/StellarConfig.java | 11 +- .../StellarTransformationTest.java | 30 ++- .../adapters/stellar/StellarAdapterTest.java | 26 +++ .../EnrichmentIntegrationTest.java | 2 + .../config/zookeeper/enrichments/test.json | 1 + .../stellar/common/utils/ConcatMap.java | 202 ++++++++++++++++++ .../common/utils/StellarProcessorUtils.java | 51 ++++- .../stellar/dsl/MapVariableResolver.java | 8 + .../metron/stellar/dsl/VariableResolver.java | 1 + .../dsl/functions/DataStructureFunctions.java | 4 + .../stellar/common/utils/ConcatMapTest.java | 83 +++++++ .../dsl/functions/BasicStellarTest.java | 18 ++ 12 files changed, 427 insertions(+), 10 deletions(-) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java create mode 100644 metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java index 62e0263d8f..9f96a228f2 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java @@ -19,6 +19,7 @@ import org.apache.metron.stellar.common.StellarAssignment; import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.VariableResolver; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -142,8 +143,14 @@ private Map getMessage( Set stellarFields { Map messageSegment = new HashMap<>(); - for(String variable : stellarFields) { - messageSegment.put(variable, message.get(variable)); + if(stellarFields.contains(VariableResolver.ALL_FIELDS)) { + //we need to include all of the fields here. + messageSegment.putAll(message); + } + else { + for (String variable : stellarFields) { + messageSegment.put(variable, message.get(variable)); + } } return messageSegment; } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index fc9184469a..3b7c7bbbf1 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -78,6 +78,32 @@ public static Collection data() { @Multiline public static String badConfig; + /** + { "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field"] + ,"config" : { + "new_field" : "MAP_GET('source.type', _)" + } + } + ] + } + */ + @Multiline + public static String configAll; + + @Test + public void testConfigAll() throws Exception { + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll)); + JSONObject input = new JSONObject(); + input.put("source.type", "test"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + Assert.assertEquals(2, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("test", input.get("new_field")); + } /** { "fieldTransformations" : [ { "transformation" : "STELLAR" @@ -92,8 +118,8 @@ public static Collection data() { ] } */ - @Multiline - public static String configRename; + @Multiline + public static String configRename; @Test public void testStellarRename() throws Exception { diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java index cd1006cfa3..e03e69ff59 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java @@ -165,6 +165,7 @@ public void test_tempVariable() throws Exception { @Multiline public static String mapConfig_default; + private void testMapEnrichment(String config, String field) throws Exception { JSONObject message = getMessage(); EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class); @@ -185,4 +186,29 @@ public void testMapEnrichment_subgroup() throws Exception { public void testMapEnrichment_default() throws Exception { testMapEnrichment(mapConfig_default, ""); } + + /** + { + "fieldMap": { + "stellar" : { + "config" : [ + "stmt1 := MAP_GET('source.type', _)" + ] + } + } + } + */ + @Multiline + public static String allVariableConfig; + + @Test + public void testAllVariableUsage() throws Exception { + JSONObject message = getMessage(); + EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(allVariableConfig, EnrichmentConfig.class); + Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar")); + ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar"); + JSONObject enriched = enrich(message, "", handler); + Assert.assertEquals("stellar_test", enriched.get("stmt1")); + } + } diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 2e22eab66b..188f18ba70 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -294,6 +294,8 @@ public static void baseValidation(Map jsonDoc) { Assert.assertEquals(1, jsonDoc.get("one")); Assert.assertEquals(1, jsonDoc.get("map.blah")); Assert.assertNotNull(jsonDoc.get("foo")); + Assert.assertNotNull(jsonDoc.get("alt_src_type")); + Assert.assertEquals("test", jsonDoc.get("alt_src_type")); Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS")); Assert.assertNotNull(jsonDoc.get("bar")); Assert.assertEquals("TEST", jsonDoc.get("bar")); diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json index 413094349c..9b997a60ac 100644 --- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json @@ -19,6 +19,7 @@ "map" : "{ 'blah' : 1}" ,"one" : "MAP_GET('blah', map)" ,"foo": "1 + 1" + ,"alt_src_type" : "MAP_GET('source.type', _)" } ,"ALL_CAPS" : "TO_UPPER(source.type)" ,"src_enrichment" : { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java new file mode 100644 index 0000000000..6282cfb888 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils; + + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ConcatMap implements Map, Serializable, KryoSerializable { + List variableMappings = new ArrayList<>(); + public ConcatMap(List variableMappings) { + this.variableMappings = variableMappings; + } + + @Override + public int size() { + int size = 0; + for(Map m : variableMappings) { + size += m.size(); + } + return size; + } + + @Override + public boolean isEmpty() { + boolean isEmpty = true; + for(Map m : variableMappings) { + isEmpty &= m.isEmpty(); + } + return isEmpty; + } + + @Override + public boolean containsKey(Object key) { + for(Map m : variableMappings) { + if(m.containsKey(key)) { + return true; + } + } + return false; + } + + @Override + public boolean containsValue(Object value) { + for(Map m : variableMappings) { + if(m.containsValue(value)) { + return true; + } + } + return false; + } + + @Override + public Object get(Object key) { + Object ret = null; + for(Map m : variableMappings) { + ret = m.get(key); + if(ret != null) { + break; + } + } + return ret; + } + + @Override + public Object put(String key, Object value) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + @Override + public Object remove(Object key) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + @Override + public void putAll(Map m) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + @Override + public void clear() { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + @Override + public Set keySet() { + Set ret = null; + for(Map m : variableMappings) { + if(ret == null) { + ret = m.keySet(); + } + else { + ret = Sets.union(ret, m.keySet()); + } + } + return ret; + } + + @Override + public Collection values() { + Collection ret = new ArrayList<>(size()); + for(Map m : variableMappings) { + ret.addAll(m.values()); + } + return ret; + } + + @Override + public Set> entrySet() { + Set> ret = null; + for(Map m : variableMappings) { + if(ret == null) { + ret = m.entrySet(); + } + else { + ret = Sets.union(ret, m.entrySet()); + } + } + return ret; + } + + @Override + public String toString() { + Iterable>> transformed = + Iterables.transform(variableMappings, x -> x.entrySet()); + Iterable> it = Iterables.filter( Iterables.concat(transformed) + , x -> x.getValue() != null + ); + return "{" + Joiner.on(", ").join(it) + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConcatMap concatMap = (ConcatMap) o; + + return variableMappings != null ? variableMappings.equals(concatMap.variableMappings) : concatMap.variableMappings == null; + + } + + @Override + public int hashCode() { + return variableMappings != null ? variableMappings.hashCode() : 0; + } + + @Override + public void write(Kryo kryo, Output output) { + int numVariableMappings = variableMappings.isEmpty()?0:variableMappings.size(); + output.writeShort(numVariableMappings); + for(Map m : variableMappings) { + byte[] b = m == null?new byte[]{}:SerDeUtils.toBytes(m); + output.writeInt(b.length); + if(b.length > 0) { + output.writeBytes(b); + } + } + } + + @Override + public void read(Kryo kryo, Input input) { + int numVariableMappings = input.readShort(); + variableMappings = new ArrayList<>(numVariableMappings); + for(int i = 0;i < numVariableMappings;++i) { + int size = input.readInt(); + if(size > 0) { + byte[] bytes = input.readBytes(size); + Map m = SerDeUtils.fromBytes(bytes, Map.class); + variableMappings.add(m); + } + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java index 4ad5a40fbf..51e7aaacd4 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java @@ -62,34 +62,59 @@ public class StellarProcessorUtils { * validates successfully and produces a result that can be serialized correctly. * * @param expression The expression to execute. - * @param variables The variables to expose to the expression. + * @param varResolver The variable resolver to use * @param context The execution context. * @return The result of executing the expression. */ - public static Object run(String expression, Map variables, Context context) { + public static Object run(String expression, VariableResolver varResolver, Context context) { validate(expression, context); - Object result = execute(expression, variables, context); + Object result = execute(expression, varResolver, context); ensureKryoSerializable(result, expression); ensureJavaSerializable(result, expression); return result; } + /** + * Execute and validate a Stellar expression. + * + *

This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @param context The execution context. + * @return The result of executing the expression. + */ + public static Object run(String expression, Map variables, Context context) { + VariableResolver varResolver = new DefaultVariableResolver( + x -> { + if(x.equals(MapVariableResolver.ALL_FIELDS)) { + return variables; + } + return variables.get(x); + } + ,x-> x.equals(MapVariableResolver.ALL_FIELDS) || variables.containsKey(x) + ); + return run(expression, varResolver, context); + } + /** * Execute a Stellar expression. * * @param expression The expression to execute. - * @param variables The variables available to the expression. + * @param variableResolver Variable Resolver to use * @param context The execution context. * @return The result of executing the expression. */ - private static Object execute(String expression, Map variables, Context context) { + private static Object execute(String expression, VariableResolver variableResolver, Context context) { StellarProcessor processor = new StellarProcessor(); + Object result = processor.parse( expression, - new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), + variableResolver, StellarFunctions.FUNCTION_RESOLVER(), context); @@ -173,6 +198,20 @@ public static Object run(String expression, Map variables) { return run(expression, variables, Context.EMPTY_CONTEXT()); } + /** + * Execute and validate a Stellar expression. + * + *

This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @return The result of executing the expression. + */ + public static Object run(String expression, VariableResolver variables) { + return run(expression, variables, Context.EMPTY_CONTEXT()); + } + /** * Execute and validate a Stellar expression. * diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java index 4c02d99f6d..872211d77e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java @@ -18,12 +18,16 @@ package org.apache.metron.stellar.dsl; + +import org.apache.metron.stellar.common.utils.ConcatMap; + import java.util.ArrayList; import java.util.List; import java.util.Map; public class MapVariableResolver implements VariableResolver { + List variableMappings = new ArrayList<>(); public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { @@ -45,6 +49,10 @@ public void add(Map... ms) { @Override public Object resolve(String variable) { + if(variable != null && variable.equals(VariableResolver.ALL_FIELDS)) { + return new ConcatMap(variableMappings); + } + for (Map variableMapping : variableMappings) { Object o = variableMapping.get(variable); if (o != null) { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java index f2624e9827..fb95d27d2e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java @@ -20,6 +20,7 @@ public interface VariableResolver { + public static final String ALL_FIELDS = "_"; Object resolve(String variable); boolean exists(String variable); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java index eb1db485d9..bebe27d5d4 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class DataStructureFunctions { @@ -211,6 +212,9 @@ public Object apply(List list) { if(o instanceof Collection) { return ((Collection)o).size(); } + else if(o instanceof Map) { + return ((Map)o).size(); + } else if(o instanceof String) { String val = (String) list.get(0); return val == null || val.isEmpty() ? 0 : val.length(); diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java new file mode 100644 index 0000000000..4c078e8cc1 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ConcatMapTest { + + @Test + public void testToString() throws Exception { + Map v1 = new HashMap<>(); + v1.put("k1", "v1"); + Map v2 = new HashMap<>(); + v2.put("k2", "v2"); + v2.put("k3", null); + Map union = new HashMap() {{ + putAll(v1); + put("k2", "v2"); + }}; + ConcatMap c = create(v1, v2); + Assert.assertEquals(c.toString(), union.toString()); + } + + private ConcatMap create(Map... ms) { + List l = new ArrayList<>(); + for(Map m : ms) { + l.add(m); + } + return new ConcatMap(l); + } + + private void assertKryoserializable(ConcatMap c) { + byte[] serialized = SerDeUtils.toBytes(c); + ConcatMap deserialized = SerDeUtils.fromBytes(serialized, ConcatMap.class); + Assert.assertEquals(deserialized, c); + } + + @Test + public void testKryoSerialization() { + Map v1 = new HashMap<>(); + v1.put("k1", "v1"); + Map v2 = new HashMap<>(); + v2.put("k2", "v2"); + v2.put("k3", null); + { + //multi maps + ConcatMap c = create(v1, v2); + assertKryoserializable(c); + } + { + //single maps + ConcatMap c = create(v1); + assertKryoserializable(c); + } + { + //empty maps + ConcatMap c = create(); + assertKryoserializable(c); + } + } + +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index aa4462a20e..ea859c5ec2 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -25,9 +25,11 @@ import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; +import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; import org.junit.Assert; import org.junit.Ignore; @@ -957,4 +959,20 @@ public void non_boolean_predicate_throws_exception() { thrown.expectMessage("The rule 'TO_UPPER(protocol)' does not return a boolean value."); runPredicate("TO_UPPER(protocol)", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))); } + + @Test + public void all_fields_test() { + final Map varMap1 = new HashMap(); + varMap1.put("field1", "val1"); + final Map varMap2 = new HashMap(); + varMap2.put("field2", "val2"); + VariableResolver resolver = new MapVariableResolver(varMap1, varMap2); + Assert.assertTrue(runPredicate("MAP_GET('field1', _) == 'val1'", resolver)); + Assert.assertTrue(runPredicate("MAP_GET('field2', _) == 'val2'", resolver)); + Assert.assertTrue(runPredicate("LENGTH(_) == 2", resolver)); + Map ret = (Map) run("_", resolver); + Assert.assertEquals(2, ret.size()); + Assert.assertEquals("val1", ret.get("field1")); + Assert.assertEquals("val2", ret.get("field2")); + } } From 4857035743c031e2343be1ea14fbccf83376f012 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 21 May 2018 18:12:07 -0400 Subject: [PATCH 2/4] Updated pcap field resolver to honor _ --- .../org/apache/metron/pcap/filter/PcapFieldResolver.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java index 4ea6836fc4..e3ac7e5766 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java @@ -18,6 +18,8 @@ package org.apache.metron.pcap.filter; +import com.google.common.collect.ImmutableList; +import org.apache.metron.stellar.common.utils.ConcatMap; import org.apache.metron.stellar.dsl.VariableResolver; import java.util.HashMap; @@ -32,6 +34,9 @@ public PcapFieldResolver(Map fieldsMap) { @Override public Object resolve(String variable) { + if(variable.equals(VariableResolver.ALL_FIELDS)) { + return new ConcatMap(ImmutableList.of(fieldsMap)); + } return fieldsMap.get(variable); } From c0afc8d11deeeae0b7bb8548bc583034060c9ab8 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 21 May 2018 18:14:00 -0400 Subject: [PATCH 3/4] Updated readme. --- metron-stellar/stellar-common/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 078799e136..4e97217071 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -37,6 +37,7 @@ For a variety of components (threat intelligence triage and field transformation The Stellar language supports the following: * Referencing fields in the enriched JSON +* Referencing all fields in the enriched JSON via the `_` reserved variable name. * String literals are quoted with either `'` or `"` * String literals support escaping for `'`, `"`, `\t`, `\r`, `\n`, and backslash * The literal `'\'foo\''` would represent `'foo'` From 2038df3c692effafc584ef32e2eb84bed905ff3f Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 22 May 2018 09:47:13 -0400 Subject: [PATCH 4/4] Updated javadocs. --- .../stellar/common/utils/ConcatMap.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java index 6282cfb888..53a52a634f 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java @@ -34,6 +34,14 @@ import java.util.Map; import java.util.Set; +/** + * ConcatMap is a lazy concatenation of a list of Maps. It is lazy in that it does not construct + * a union of all of the maps, but rather keeps the maps separate. Key/Value resolution is + * done via a first-wins strategy (i.e. the first map which has a key will be used). + * + * Also, note, that this is an immutable map, so operations which require mutation will have + * UnsupportedOperationException thrown. + */ public class ConcatMap implements Map, Serializable, KryoSerializable { List variableMappings = new ArrayList<>(); public ConcatMap(List variableMappings) { @@ -58,6 +66,11 @@ public boolean isEmpty() { return isEmpty; } + /** + * If any maps contains the key, then this will return true. + * @param key + * @return + */ @Override public boolean containsKey(Object key) { for(Map m : variableMappings) { @@ -68,6 +81,12 @@ public boolean containsKey(Object key) { return false; } + /** + * + * If any maps contains the value, then this will return true. + * @param value + * @return + */ @Override public boolean containsValue(Object value) { for(Map m : variableMappings) { @@ -78,6 +97,11 @@ public boolean containsValue(Object value) { return false; } + /** + * The first map which contains the key will have the associated value returned. + * @param key + * @return + */ @Override public Object get(Object key) { Object ret = null; @@ -90,21 +114,42 @@ public Object get(Object key) { return ret; } + /** + * This is an immutable map and this operation is not supported. + * @param key + * @param value + * @return + */ @Override public Object put(String key, Object value) { throw new UnsupportedOperationException("Merged map is immutable."); } + /** + * + * This is an immutable map and this operation is not supported. + * @param key + * @return + */ @Override public Object remove(Object key) { throw new UnsupportedOperationException("Merged map is immutable."); } + /** + * + * This is an immutable map and this operation is not supported. + * @param m + */ @Override public void putAll(Map m) { throw new UnsupportedOperationException("Merged map is immutable."); } + /** + * + * This is an immutable map and this operation is not supported. + */ @Override public void clear() { throw new UnsupportedOperationException("Merged map is immutable."); @@ -124,6 +169,10 @@ public Set keySet() { return ret; } + /** + * Note: this makes a copy of the values, so it is not fundamentally lazy. + * @return + */ @Override public Collection values() { Collection ret = new ArrayList<>(size()); @@ -133,6 +182,11 @@ public Collection values() { return ret; } + /** + * This is a lazy entry collection of the associated maps. If there are duplicate keys, they will appear + * twice here, so be careful. + * @return + */ @Override public Set> entrySet() { Set> ret = null;