From 290bc793a4cecb1c7c83ef4cfb77f67f5ef7dbbe Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 5 Sep 2018 12:12:51 -0400 Subject: [PATCH 1/6] METRON-1717 Renamed Storm Profiler package --- .../src/main/flux/profiler/remote.yaml | 10 +++++----- .../{bolt => storm}/FixedFrequencyFlushSignal.java | 2 +- .../metron/profiler/{bolt => storm}/FlushSignal.java | 2 +- .../profiler/{bolt => storm}/HBaseEmitter.java | 2 +- .../profiler/{bolt => storm}/KafkaEmitter.java | 2 +- .../profiler/{bolt => storm}/ManualFlushSignal.java | 2 +- .../profiler/{bolt => storm}/ProfileBuilderBolt.java | 12 ++++++------ .../profiler/{bolt => storm}/ProfileHBaseMapper.java | 2 +- .../{bolt => storm}/ProfileMeasurementEmitter.java | 2 +- .../{bolt => storm}/ProfileSplitterBolt.java | 6 +----- .../FixedFrequencyFlushSignalTest.java | 2 +- .../profiler/{bolt => storm}/HBaseEmitterTest.java | 4 +--- .../profiler/{bolt => storm}/KafkaEmitterTest.java | 2 +- .../{bolt => storm}/ProfileBuilderBoltTest.java | 2 +- .../{bolt => storm}/ProfileHBaseMapperTest.java | 4 ++-- .../{bolt => storm}/ProfileSplitterBoltTest.java | 2 +- 16 files changed, 26 insertions(+), 32 deletions(-) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/FixedFrequencyFlushSignal.java (98%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/FlushSignal.java (97%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/HBaseEmitter.java (98%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/KafkaEmitter.java (99%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/ManualFlushSignal.java (97%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/ProfileBuilderBolt.java (97%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/ProfileHBaseMapper.java (98%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/ProfileMeasurementEmitter.java (97%) rename metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/{bolt => storm}/ProfileSplitterBolt.java (97%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/FixedFrequencyFlushSignalTest.java (97%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/HBaseEmitterTest.java (97%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/KafkaEmitterTest.java (99%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/ProfileBuilderBoltTest.java (99%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/ProfileHBaseMapperTest.java (98%) rename metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/{bolt => storm}/ProfileSplitterBoltTest.java (99%) diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index 2f40554232..da71b273b1 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -44,7 +44,7 @@ components: - "${profiler.hbase.column.family}" - id: "hbaseMapper" - className: "org.apache.metron.profiler.bolt.ProfileHBaseMapper" + className: "org.apache.metron.profiler.storm.ProfileHBaseMapper" properties: - name: "rowKeyBuilder" ref: "rowKeyBuilder" @@ -113,10 +113,10 @@ components: args: [ref: "kafkaWriterProps"] - id: "kafkaEmitter" - className: "org.apache.metron.profiler.bolt.KafkaEmitter" + className: "org.apache.metron.profiler.storm.KafkaEmitter" - id: "hbaseEmitter" - className: "org.apache.metron.profiler.bolt.HBaseEmitter" + className: "org.apache.metron.profiler.storm.HBaseEmitter" - id: "windowDuration" className: "org.apache.storm.topology.base.BaseWindowedBolt$Duration" @@ -140,12 +140,12 @@ spouts: bolts: - id: "splitterBolt" - className: "org.apache.metron.profiler.bolt.ProfileSplitterBolt" + className: "org.apache.metron.profiler.storm.ProfileSplitterBolt" constructorArgs: - "${kafka.zk}" - id: "builderBolt" - className: "org.apache.metron.profiler.bolt.ProfileBuilderBolt" + className: "org.apache.metron.profiler.storm.ProfileBuilderBolt" configMethods: - name: "withZookeeperUrl" args: ["${kafka.zk}"] diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java similarity index 98% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java index 8c0a0b1482..02503c2699 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java @@ -17,7 +17,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FlushSignal.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java similarity index 97% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FlushSignal.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java index 0a9fc76c87..f1e66c2f69 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FlushSignal.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java @@ -17,7 +17,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; /** * Signals when it is time to flush a profile. diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java similarity index 98% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java index e4e35528a4..0d1b806f91 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java @@ -16,7 +16,7 @@ * limitations under the License. * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.storm.task.OutputCollector; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaEmitter.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java similarity index 99% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaEmitter.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java index 87920da400..af1fbcabf9 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaEmitter.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java @@ -17,7 +17,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ClassUtils; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ManualFlushSignal.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java similarity index 97% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ManualFlushSignal.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java index d8e9539a52..81feb8e72e 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ManualFlushSignal.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java @@ -16,7 +16,7 @@ * limitations under the License. * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; /** * Signals that a flush should occur. diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java similarity index 97% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java index f9c0edd72d..205159ed7b 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.commons.collections4.CollectionUtils; import org.apache.curator.RetryPolicy; @@ -62,10 +62,10 @@ import java.util.concurrent.TimeUnit; import static java.lang.String.format; -import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.ENTITY_TUPLE_FIELD; -import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.MESSAGE_TUPLE_FIELD; -import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.PROFILE_TUPLE_FIELD; -import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.ENTITY_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.MESSAGE_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.PROFILE_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD; /** * A Storm bolt that is responsible for building a profile. @@ -361,7 +361,7 @@ private void handleMessage(Tuple input) { // keep track of time activeFlushSignal.update(timestamp); - + // distribute the message MessageRoute route = new MessageRoute(definition, entity, message, timestamp); synchronized (messageDistributor) { diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java similarity index 98% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java index 5448e39df7..f36496cebe 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.hbase.bolt.mapper.ColumnList; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileMeasurementEmitter.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java similarity index 97% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileMeasurementEmitter.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java index e1fe4e1501..4ad4135d92 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileMeasurementEmitter.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java @@ -16,7 +16,7 @@ * limitations under the License. * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.storm.task.OutputCollector; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java similarity index 97% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java rename to metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java index f73a9f1b98..81179b6020 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java @@ -18,16 +18,13 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.profiler.DefaultMessageRouter; import org.apache.metron.profiler.MessageRoute; import org.apache.metron.profiler.MessageRouter; -import org.apache.metron.profiler.clock.Clock; -import org.apache.metron.profiler.clock.ClockFactory; -import org.apache.metron.profiler.clock.DefaultClockFactory; import org.apache.metron.stellar.dsl.Context; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -45,7 +42,6 @@ import java.lang.invoke.MethodHandles; import java.util.List; import java.util.Map; -import java.util.Optional; /** * The Storm bolt responsible for filtering incoming messages and directing diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java similarity index 97% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java index b8949c5c6f..8b8813bf4b 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java @@ -17,7 +17,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.junit.Test; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/HBaseEmitterTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java similarity index 97% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/HBaseEmitterTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java index 35ca4d9788..2f9eca4654 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/HBaseEmitterTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.profiler.ProfileConfig; @@ -26,14 +26,12 @@ import org.apache.metron.profiler.ProfileMeasurement; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Values; -import org.json.simple.JSONObject; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.io.IOException; -import java.util.Collections; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaEmitterTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java similarity index 99% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaEmitterTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java index 95a2d29475..51ca3a4972 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaEmitterTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java similarity index 99% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java index 3132ae62df..44556646d1 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.configuration.profiler.ProfilerConfigurations; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java similarity index 98% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java index 04c774c34d..f623d38a51 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.configuration.profiler.ProfileResult; @@ -90,4 +90,4 @@ public void testExpiresUndefined() throws Exception { Optional actual = mapper.getTTL(tuple); Assert.assertFalse(actual.isPresent()); } -} \ No newline at end of file +} diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java similarity index 99% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java rename to metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java index c879b4bdab..93d2ac4617 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.bolt; +package org.apache.metron.profiler.storm; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.profiler.ProfileConfig; From b88c0e72974480750255d6e64faed24cf876527b Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 5 Sep 2018 13:15:46 -0400 Subject: [PATCH 2/6] Rename package to org.apache.metron.profiler.storm --- .../.gitignore | 0 .../README.md | 0 .../pom.xml | 0 .../src/main/assembly/assembly.xml | 0 .../src/main/config/profiler.properties | 0 .../src/main/flux/profiler/remote.yaml | 0 .../storm/FixedFrequencyFlushSignal.java | 0 .../metron/profiler/storm/FlushSignal.java | 0 .../metron/profiler/storm/HBaseEmitter.java | 0 .../metron/profiler/storm/KafkaEmitter.java | 0 .../profiler/storm/ManualFlushSignal.java | 0 .../profiler/storm/ProfileBuilderBolt.java | 0 .../profiler/storm/ProfileHBaseMapper.java | 0 .../storm/ProfileMeasurementEmitter.java | 0 .../profiler/storm/ProfileSplitterBolt.java | 0 .../src/main/resources/META-INF/LICENSE | 0 .../src/main/resources/META-INF/NOTICE | 0 .../main/scripts/start_profiler_topology.sh | 0 .../zookeeper/event-time-test/profiler.json | 0 .../processing-time-test/profiler.json | 0 .../profile-with-stats/profiler.json | 0 .../integration/ConfigUploadComponent.java | 0 .../profiler/integration/MessageBuilder.java | 0 .../integration/ProfilerIntegrationTest.java | 20 ++----------------- .../storm/FixedFrequencyFlushSignalTest.java | 0 .../profiler/storm/HBaseEmitterTest.java | 0 .../profiler/storm/KafkaEmitterTest.java | 0 .../storm/ProfileBuilderBoltTest.java | 0 .../storm/ProfileHBaseMapperTest.java | 0 .../storm/ProfileSplitterBoltTest.java | 0 .../src/test/resources/log4j.properties | 0 metron-analytics/pom.xml | 2 +- .../packaging/docker/deb-docker/pom.xml | 2 +- .../packaging/docker/rpm-docker/pom.xml | 2 +- .../ZKConfigurationsCacheIntegrationTest.java | 2 +- 35 files changed, 6 insertions(+), 22 deletions(-) rename metron-analytics/{metron-profiler => metron-profiler-storm}/.gitignore (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/README.md (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/pom.xml (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/assembly/assembly.xml (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/config/profiler.properties (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/flux/profiler/remote.yaml (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/resources/META-INF/LICENSE (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/resources/META-INF/NOTICE (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/main/scripts/start_profiler_topology.sh (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/config/zookeeper/event-time-test/profiler.json (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/config/zookeeper/processing-time-test/profiler.json (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/config/zookeeper/profile-with-stats/profiler.json (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java (94%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java (100%) rename metron-analytics/{metron-profiler => metron-profiler-storm}/src/test/resources/log4j.properties (100%) diff --git a/metron-analytics/metron-profiler/.gitignore b/metron-analytics/metron-profiler-storm/.gitignore similarity index 100% rename from metron-analytics/metron-profiler/.gitignore rename to metron-analytics/metron-profiler-storm/.gitignore diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler-storm/README.md similarity index 100% rename from metron-analytics/metron-profiler/README.md rename to metron-analytics/metron-profiler-storm/README.md diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml similarity index 100% rename from metron-analytics/metron-profiler/pom.xml rename to metron-analytics/metron-profiler-storm/pom.xml diff --git a/metron-analytics/metron-profiler/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-storm/src/main/assembly/assembly.xml similarity index 100% rename from metron-analytics/metron-profiler/src/main/assembly/assembly.xml rename to metron-analytics/metron-profiler-storm/src/main/assembly/assembly.xml diff --git a/metron-analytics/metron-profiler/src/main/config/profiler.properties b/metron-analytics/metron-profiler-storm/src/main/config/profiler.properties similarity index 100% rename from metron-analytics/metron-profiler/src/main/config/profiler.properties rename to metron-analytics/metron-profiler-storm/src/main/config/profiler.properties diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml similarity index 100% rename from metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml rename to metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignal.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/FlushSignal.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/HBaseEmitter.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ManualFlushSignal.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileMeasurementEmitter.java diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java similarity index 100% rename from metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java rename to metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java diff --git a/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE b/metron-analytics/metron-profiler-storm/src/main/resources/META-INF/LICENSE similarity index 100% rename from metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE rename to metron-analytics/metron-profiler-storm/src/main/resources/META-INF/LICENSE diff --git a/metron-analytics/metron-profiler/src/main/resources/META-INF/NOTICE b/metron-analytics/metron-profiler-storm/src/main/resources/META-INF/NOTICE similarity index 100% rename from metron-analytics/metron-profiler/src/main/resources/META-INF/NOTICE rename to metron-analytics/metron-profiler-storm/src/main/resources/META-INF/NOTICE diff --git a/metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh b/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh similarity index 100% rename from metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh rename to metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/event-time-test/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json similarity index 100% rename from metron-analytics/metron-profiler/src/test/config/zookeeper/event-time-test/profiler.json rename to metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/processing-time-test/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json similarity index 100% rename from metron-analytics/metron-profiler/src/test/config/zookeeper/processing-time-test/profiler.json rename to metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/profile-with-stats/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json similarity index 100% rename from metron-analytics/metron-profiler/src/test/config/zookeeper/profile-with-stats/profiler.json rename to metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java similarity index 94% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java index 268ce26309..8f7741d352 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java @@ -20,12 +20,7 @@ package org.apache.metron.profiler.integration; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import com.esotericsoftware.kryo.serializers.FieldSerializer; import org.adrianwalker.multilinestring.Multiline; -import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; @@ -45,26 +40,16 @@ import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.storm.Config; -import org.apache.storm.serialization.KryoTupleDeserializer; -import org.apache.storm.serialization.KryoTupleSerializer; -import org.apache.storm.serialization.KryoValuesDeserializer; -import org.apache.storm.serialization.KryoValuesSerializer; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.TupleImpl; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.Mockito; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -74,7 +59,6 @@ import static com.google.code.tempusfugit.temporal.WaitFor.waitOrTimeout; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -82,8 +66,8 @@ */ public class ProfilerIntegrationTest extends BaseIntegrationTest { - private static final String TEST_RESOURCES = "../../metron-analytics/metron-profiler/src/test"; - private static final String FLUX_PATH = "../metron-profiler/src/main/flux/profiler/remote.yaml"; + private static final String TEST_RESOURCES = "../../metron-analytics/metron-profiler-storm/src/test"; + private static final String FLUX_PATH = "src/main/flux/profiler/remote.yaml"; public static final long startAt = 10; public static final String entity = "10.0.0.1"; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/FixedFrequencyFlushSignalTest.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/HBaseEmitterTest.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileHBaseMapperTest.java diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java similarity index 100% rename from metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java diff --git a/metron-analytics/metron-profiler/src/test/resources/log4j.properties b/metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties similarity index 100% rename from metron-analytics/metron-profiler/src/test/resources/log4j.properties rename to metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml index 37ee2b0ede..bcfc6e0e64 100644 --- a/metron-analytics/pom.xml +++ b/metron-analytics/pom.xml @@ -43,7 +43,7 @@ metron-maas-service metron-maas-common metron-statistics - metron-profiler + metron-profiler-storm metron-profiler-client metron-profiler-common metron-profiler-spark diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml index a0df09abb0..92d63cc989 100644 --- a/metron-deployment/packaging/docker/deb-docker/pom.xml +++ b/metron-deployment/packaging/docker/deb-docker/pom.xml @@ -126,7 +126,7 @@ - ${metron_dir}/metron-analytics/metron-profiler/target/ + ${metron_dir}/metron-analytics/metron-profiler-storm/target/ *.tar.gz diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 0d9c4d153b..2220ca8b2d 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -162,7 +162,7 @@ - ${metron_dir}/metron-analytics/metron-profiler/target/ + ${metron_dir}/metron-analytics/metron-profiler-storm/target/ *.tar.gz diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java index 5240d7ac18..a7dc248113 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java @@ -119,7 +119,7 @@ public class ZKConfigurationsCacheIntegrationTest { @Multiline public static String globalConfig; - public static File profilerDir = new File("../../metron-analytics/metron-profiler/src/test/config/zookeeper"); + public static File profilerDir = new File("../../metron-analytics/metron-profiler-storm/src/test/config/zookeeper"); public ConfigurationsCache cache; public ZKServerComponent zkComponent; From 27e69d41c2e8a982dca23dfc6feca737b0e48c12 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 5 Sep 2018 16:36:26 -0400 Subject: [PATCH 3/6] Updated READMEs --- README.md | 2 +- .../metron-profiler-common/README.md | 386 +++++++++++ .../metron-profiler-repl/README.md | 7 +- .../metron-profiler-spark/README.md | 16 +- .../metron-profiler-storm/README.md | 648 ++---------------- 5 files changed, 472 insertions(+), 587 deletions(-) create mode 100644 metron-analytics/metron-profiler-common/README.md diff --git a/README.md b/README.md index 485880777a..95b6fafa75 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ Some useful utilities that cross all of these parts of the architecture: * [Stellar](metron-platform/metron-common) : A custom data transformation language that is used throughout metron from simple field transformation to expressing triage rules. * [Model as a Service](metron-analytics/metron-maas-service) : A Yarn application which can deploy machine learning and statistical models onto the cluster along with the associated Stellar functions to be able to call out to them in a scalable manner. * [Data management](metron-platform/metron-data-management) : A set of data management utilities aimed at getting data into HBase in a format which will allow data flowing through metron to be enriched with the results. Contains integrations with threat intelligence feeds exposed via TAXII as well as simple flat file structures. -* [Profiler](metron-analytics/metron-profiler) : A feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. +* [Profiler](metron-analytics/metron-profiler-common) : A feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. # Notes on Adding a New Sensor In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x, diff --git a/metron-analytics/metron-profiler-common/README.md b/metron-analytics/metron-profiler-common/README.md new file mode 100644 index 0000000000..1f60014c6f --- /dev/null +++ b/metron-analytics/metron-profiler-common/README.md @@ -0,0 +1,386 @@ + +# Metron Profiler + +* [Introduction](#introduction) +* [Getting Started](#getting-started) +* [Profiles](#profiles) +* [Examples](#examples) + +## Introduction + +The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. + +This is achieved by summarizing the telemetry data consumed by Metron over tumbling windows. A summary statistic is applied to the data received within a given window. Collecting these values across many windows result in a time series that is useful for analysis. + +Any field contained within a message can be used to generate a profile. A profile can even be produced by combining fields that originate in different data sources. A user has considerable power to transform the data used in a profile by leveraging the Stellar language. + +There are three separate ports of the Profiler that share this common code base. +* The [Storm Profiler](../metron-profiler-storm/README.md) builds low-latency profiles over streaming data sets. +* The [Spark Profiler](../metron-profiler-spark/README.md) backfills profiles using archived telemetry. +* The [REPL Profiler](../metron-profiler-repl/README.md) allows profiles to be tested and debugged within the Stellar REPL. + +## Getting Started + +1. [Create a profile](../metron-profiler-repl/README.md#getting-started) using the Stellar REPL. Validate your profile using mock data, then apply real, live data. + +1. [Backfill your profile](../metron-profiler-spark/README.md#getting-started) using archived telemetry to see how your profile behaves over time. + +1. [Deploy your profile](../metron-profiler-storm/README.md#getting-started) to Storm to maintain a low-latency profile over a streaming data set. + +1. [Retrieve your profile data](../metron-profiler-client/README.md) using the Stellar API so that you can build enrichments, alert on abnormalities + +1. Explore more ways to create [profiles](#more-examples). + +## Profiles + +Let's start with a simple example. The following profile maintains a count of the number of telemetry messages for each IP source address. A counter is initialized to 0, then incremented each time a message is received for a give IP source address. At regular intervals the count is flushed and stored. Over time this results in a time series describing the amount of telemetry received for each IP source address. + +``` +{ + "profiles": [ + { + "profile": "hello-world", + "foreach": "ip_src_addr", + "init": { + "count": 0 + }, + "update": { + "count": "count + 1" + }, + "result": "count", + } + ] +} +``` + +A profile definition contains two fields; only one of which is required. + +``` +{ + "profiles": [ + { "profile": "one", ... }, + { "profile": "two", ... } + ], + "timestampField": "timestamp" +} +``` + +| Name | | Description +|--- |--- |--- +| [profiles](#profiles) | Required | A list of zero or more Profile definitions. +| [timestampField](#timestampfield) | Optional | Indicates whether processing time or event time should be used. By default, processing time is enabled. + + +#### `profiles` + +*Required* + +A list of zero or more Profile definitions. + +#### `timestampField` + +*Optional* + +Indicates whether processing time or event time is used. By default, processing time is enabled. + +##### Processing Time + +By default, no `timestampField` is defined. In this case, the Profiler uses system time when generating profiles. This means that the profiles are generated based on when the data has been processed by the Profiler. This is also known as 'processing time'. + +This is the simplest mode of operation, but has some draw backs. If the Profiler is consuming live data and all is well, the processing and event times will likely remain similar and consistent. If processing time diverges from event time, then the Profiler will generate skewed profiles. + +There are a few scenarios that might cause skewed profiles when using processing time. For example when a system has undergone a scheduled maintenance window and is restarted, a high volume of messages will need to be processed by the Profiler. The output of the Profiler might indicate an increase in activity during this time, although no change in activity actually occurred on the target network. The same situation could occur if an upstream system which provides telemetry undergoes an outage. + +[Event Time](#event-time) can be used to mitigate these problems. + +##### Event Time + +Alternatively, a `timestampField` can be defined. This must be the name of a field contained within the telemetry processed by the Profiler. The Profiler will extract and use the timestamp contained within this field. + +* If a message does not contain this field, it will be dropped. + +* The field must contain a timestamp in epoch milliseconds expressed as either a numeric or string. Otherwise, the message will be dropped. + +* The Profiler will use the same field across all telemetry sources and for all profiles. + +* Be aware of clock skew across telemetry sources. If your profile is processing telemetry from multiple sources where the clock differs significantly, the Profiler may assume that some of those messages are late and will be ignored. Adjusting the [`profiler.window.duration`](#profilerwindowduration) and [`profiler.window.lag`](#profilerwindowlag) can help accommodate skewed clocks. + +### Profiles + +A profile definition requires a JSON-formatted set of elements, many of which can contain Stellar code. The specification contains the following elements. (For the impatient, skip ahead to the [Examples](#examples).) + +| Name | | Description +|--- |--- |--- +| [profile](#profile) | Required | Unique name identifying the profile. +| [foreach](#foreach) | Required | A separate profile is maintained "for each" of these. +| [onlyif](#onlyif) | Optional | Boolean expression that determines if a message should be applied to the profile. +| [groupBy](#groupby) | Optional | One or more Stellar expressions used to group the profile measurements when persisted. +| [init](#init) | Optional | One or more expressions executed at the start of a window period. +| [update](#update) | Required | One or more expressions executed when a message is applied to the profile. +| [result](#result) | Required | Stellar expressions that are executed when the window period expires. +| [expires](#expires) | Optional | Profile data is purged after this period of time, specified in days. + +#### `profile` + +*Required* + +A unique name identifying the profile. The field is treated as a string. + +#### `foreach` + +*Required* + +A separate profile is maintained 'for each' of these. This is effectively the entity that the profile is describing. The field is expected to contain a Stellar expression whose result is the entity name. + +For example, if `ip_src_addr` then a separate profile would be maintained for each unique IP source address in the data; 10.0.0.1, 10.0.0.2, etc. + +#### `onlyif` + +*Optional* + +An expression that determines if a message should be applied to the profile. A Stellar expression that returns a Boolean is expected. A message is only applied to a profile if this expression is true. This allows a profile to filter the messages that get applied to it. + +#### `groupBy` + +*Optional* + +One or more Stellar expressions used to group the profile measurements when persisted. This can be used to sort the Profile data to allow for a contiguous scan when accessing subsets of the data. This is also one way to deal with calendar effects. For example, where activity on a weekday can be very different from a weekend. + +A common use case would be grouping by day of week. This allows a contiguous scan to access all profile data for Mondays only. Using the following definition would achieve this. + +``` +"groupBy": [ "DAY_OF_WEEK(start)" ] +``` + +The expression can reference any of these variables. +* Any variable defined by the profile in its `init` or `update` expressions. +* `profile` The name of the profile. +* `entity` The name of the entity being profiled. +* `start` The start time of the profile period in epoch milliseconds. +* `end` The end time of the profile period in epoch milliseconds. +* `duration` The duration of the profile period in milliseconds. +* `result` The result of executing the `result` expression. + +#### `init` + +*Optional* + +One or more expressions executed at the start of a window period. A map is expected where the key is the variable name and the value is a Stellar expression. The map can contain zero or more variable:expression pairs. At the start of each window period, each expression is executed once and stored in the given variable. Note that constant init values such as "0" must be in quotes regardless of their type, as the init value must be a string to be executed by Stellar. + +``` +"init": { + "var1": "0", + "var2": "1" +} +``` + +#### `update` + +*Required* + +One or more expressions executed when a message is applied to the profile. A map is expected where the key is the variable name and the value is a Stellar expression. The map can include 0 or more variables/expressions. When each message is applied to the profile, the expression is executed and stored in a variable with the given name. + +``` +"update": { + "var1": "var1 + 1", + "var2": "var2 + 1" +} +``` + +#### `result` + +*Required* + +Stellar expressions that are executed when the window period expires. The expressions are expected to summarize the messages that were applied to the profile over the window period. In the most basic form a single result is persisted for later retrieval. +``` +"result": "var1 + var2" +``` + +For more advanced use cases, a profile can generate two types of results. A profile can define one or both of these result types at the same time. +* `profile`: A required expression that defines a value that is persisted for later retrieval. +* `triage`: An optional expression that defines values that are accessible within the Threat Triage process. + +**profile** + +A required Stellar expression that results in a value that is persisted in the profile store for later retrieval. The expression can result in any object that is Kryo serializable. These values can be retrieved for later use with the [Profiler Client](../metron-profiler-client). +``` +"result": { + "profile": "2 + 2" +} +``` + +An alternative, simplified form is also acceptable. +``` +"result": "2 + 2" +``` + +**triage** + +An optional map of one or more Stellar expressions. The value of each expression is made available to the Threat Triage process under the given name. Each expression must result in a either a primitive type, like an integer, long, or short, or a String. All other types will result in an error. + +In the following example, three values, the minimum, the maximum and the mean are appended to a message. This message is consumed by Metron, like other sources of telemetry, and each of these values are accessible from within the Threat Triage process using the given field names; `min`, `max`, and `mean`. +``` +"result": { + "triage": { + "min": "STATS_MIN(stats)", + "max": "STATS_MAX(stats)", + "mean": "STATS_MEAN(stats)" + } +} +``` + +#### `expires` + +*Optional* + +A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire. + +The REPL can be a powerful tool for developing profiles. Read all about [Developing Profiles](../metron-profiler-client/#developing_profiles). + +## Examples + +The following examples are intended to highlight the functionality provided by the Profiler. Try out these examples easily in the Stellar REPL as described in the [Getting Started](#getting-started) section. + +### Example 1 + +This example captures the ratio of DNS traffic to HTTP traffic for each host. The following configuration would be used to generate this profile. + +``` +{ + "profiles": [ + { + "profile": "dns-to-http-by-source", + "foreach": "ip_src_addr", + "onlyif": "protocol == 'DNS' or protocol == 'HTTP'", + "init": { + "num_dns": 1.0, + "num_http": 1.0 + }, + "update": { + "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)", + "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)" + }, + "result": "num_dns / num_http" + } + ] +} +``` + +This creates a profile... + * Named ‘dns-to-http-by-source’ + * That for each IP source address + * Only if the 'protocol' field equals 'HTTP' or 'DNS' + * Accumulates the number of DNS requests + * Accumulates the number of HTTP requests + * Returns the ratio of these as the result + +### Example 2 + +This example captures the average of the `length` field for HTTP traffic. The following profile could be used. + +``` +{ + "profiles": [ + { + "profile": "avg-http-length", + "foreach": "ip_src_addr", + "onlyif": "protocol == 'HTTP'", + "update": { "s": "STATS_ADD(s, length)" }, + "result": "STATS_MEAN(s)" + } + ] +} +``` + +This creates a profile... + * Named ‘avg-http-length’ + * That for each IP source address + * Only if the 'protocol' field is 'HTTP' + * Captures the `length` field + * Calculates the average as the result + +It is important to note that the Profiler can persist any serializable Object, not just numeric values. Instead of storing the actual mean, the profile could store a statistical sketch of the lengths. This summary can then be used at a later time to calculate the mean, min, max, percentiles, or any other sensible metric. This provides a much greater degree of flexibility. The following Stellar REPL session shows how you might do this. + +1. Retrieve the last 30 minutes of profile measurements for a specific host. + ``` + $ source /etc/default/metron + $ bin/stellar -z $ZOOKEEPER + + [Stellar]>>> stats := PROFILE_GET( "example4", "10.0.0.1", PROFILE_FIXED(30, "MINUTES")) + [org.apache.metron.common.math.stats.OnlineStatisticsProvider@79fe4ab9, ...] + ``` + +1. Calculate different summary metrics using the same profile data. + ``` + [Stellar]>>> aStat := GET_FIRST(stats) + org.apache.metron.common.math.stats.OnlineStatisticsProvider@79fe4ab9 + + [Stellar]>>> STATS_MEAN(aStat) + 15979.0625 + + [Stellar]>>> STATS_PERCENTILE(aStat, 90) + 30310.958 + ``` + +1. Merge all of the profile measurements over the past 30 minutes into a single sketch and calculate the 90th percentile. + ``` + [Stellar]>>> merged := STATS_MERGE( stats) + + [Stellar]>>> STATS_PERCENTILE(merged, 90) + 29810.992 + ``` + + +More information on accessing profile data can be found in the [Profiler Client](../metron-profiler-client/README.md). + +More information on using the [`STATS_*` functions](../metron-statistics/README.md). + + +### Example 3 + +This profile captures the vertex degree of a host. If you view network communication as a directed graph, the in and out degree of each host can distinguish behaviors. Anomalies can serve as an indicator of compromise. For example, you might find clients normally have an out-degree >> in-degree, whereas a server might be the opposite. + +``` +{ + "profiles": [ + { + "profile": "in-degrees", + "onlyif": "source.type == 'yaf'", + "foreach": "ip_dst_addr", + "update": { "in": "HLLP_ADD(in, ip_src_addr)" }, + "result": "HLLP_CARDINALITY(in)" + }, + { + "profile": "out-degrees", + "onlyif": "source.type == 'yaf'", + "foreach": "ip_src_addr", + "update": { "out": "HLLP_ADD(out, ip_dst_addr)" }, + "result": "HLLP_CARDINALITY(out)" + } + ] +} +``` + +This creates a profile... + * Named ‘in-degrees’ + * That for each IP destination address + * Captures the IP source address + * Then calculates the cardinality; the number of unique IPs this host has interacted with + +The second profile calculates the out-degree. diff --git a/metron-analytics/metron-profiler-repl/README.md b/metron-analytics/metron-profiler-repl/README.md index e0f61d41cc..128ed2fff0 100644 --- a/metron-analytics/metron-profiler-repl/README.md +++ b/metron-analytics/metron-profiler-repl/README.md @@ -17,7 +17,8 @@ limitations under the License. --> # Metron Profiler for the Stellar REPL -This project allows profiles to be executed within the Stellar REPL. This is a port of the Profiler to run in the Stellar REPL. +This project allows profiles to be executed within the Stellar REPL. This is a port of the Profiler to the Stellar REPL that allows profiles to be tested and debugged within a controlled environment. + * [Introduction](#introduction) * [Getting Started](#getting-started) @@ -27,7 +28,7 @@ This project allows profiles to be executed within the Stellar REPL. This is a p Creating and refining profiles is an iterative process. Iterating against a live stream of data is slow, difficult and error prone. Running the Profiler in the Stellar REPL provides a controlled and isolated execution environment to create, refine and troubleshoot profiles. -For an introduction to the Profiler, see the [Profiler README](../metron-profiler/README.md). +For an introduction to the Profiler, see the [Profiler README](../metron-profiler-common/README.md). ## Getting Started @@ -132,7 +133,7 @@ This section describes how to get started using the Profiler in the Stellar REPL Profiler{1 profile(s), 10 messages(s), 10 route(s)} ``` -1. After you are satisfied with your profile, the next step is to deploy the profile against the live stream of telemetry being capture by Metron. This involves deploying the profile to either the [Storm Profiler](../metron-profiler/README.md) or the [Spark Profiler](../metron-profiler-spark/README.md). +1. After you are satisfied with your profile, the next step is to deploy the profile against the live stream of telemetry being capture by Metron. This involves deploying the profile to either the [Storm Profiler](../metron-profiler-storm/README.md) or the [Spark Profiler](../metron-profiler-spark/README.md). ## Installation diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md index 0a31263fb1..d137e51924 100644 --- a/metron-analytics/metron-profiler-spark/README.md +++ b/metron-analytics/metron-profiler-spark/README.md @@ -17,7 +17,7 @@ limitations under the License. --> # Metron Profiler for Spark -This project allows profiles to be executed using [Apache Spark](https://spark.apache.org). This is a port of the Profiler to Spark. +This project allows profiles to be executed using [Apache Spark](https://spark.apache.org). This is a port of the Profiler to Spark that allows you to backfill profiles using archived telemetry. * [Introduction](#introduction) * [Getting Started](#getting-started) @@ -27,9 +27,9 @@ This project allows profiles to be executed using [Apache Spark](https://spark.a ## Introduction -Using the [Streaming Profiler](../metron-profiler/README.md) in [Apache Storm](http://storm.apache.org) allows you to create profiles based on the stream of telemetry being captured, enriched, triaged, and indexed by Metron. This does not allow you to create a profile based on telemetry that was captured in the past. +Using the [Streaming Profiler](../metron-profiler-storm/README.md) in [Apache Storm](http://storm.apache.org) allows you to create profiles based on the stream of telemetry being captured, enriched, triaged, and indexed by Metron. This does not allow you to create a profile based on telemetry that was captured in the past. -There are many cases where you might want to produce a profile from telemetry in the past. This is referred to as "profile seeding". +There are many cases where you might want to produce a profile from telemetry in the past. This is referred to as profile seeding or backfilling. * As a Security Data Scientist, I want to understand the historical behaviors and trends of a profile so that I can determine if the profile has predictive value for model building. @@ -39,12 +39,10 @@ The Batch Profiler running in [Apache Spark](https://spark.apache.org) allows yo The portion of a profile produced by the Batch Profiler should be indistinguishable from the portion created by the Streaming Profiler. Consumers of the profile should not care how the profile was generated. Using the Streaming Profiler together with the Batch Profiler allows you to create a complete profile over a wide range of time. -For an introduction to the Profiler and Profiler concepts, see the [Profiler README](../metron-profiler/README.md). +For an introduction to the Profiler, see the [Profiler README](../metron-profiler-common/README.md). ## Getting Started - - 1. Create a profile definition by editing `$METRON_HOME/config/zookeeper/profiler.json` as follows. ``` @@ -95,8 +93,6 @@ The Batch Profiler package is installed automatically when installing Metron usi The Batch Profiler requires Spark version 2.3.0+. -### Packages - #### Build the RPM 1. Build Metron. @@ -115,7 +111,7 @@ The Batch Profiler requires Spark version 2.3.0+. find ./ -name "metron-profiler-spark*.rpm" ``` -### Build the DEB +#### Build the DEB 1. Build Metron. ``` @@ -151,7 +147,7 @@ You can store both settings for the Profiler along with settings for Spark in th ### `profiler.batch.input.path` -*Default*: hdfs://localhost:9000/apps/metron/indexing/indexed/*/* +*Default*: "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*" The path to the input data read by the Batch Profiler. diff --git a/metron-analytics/metron-profiler-storm/README.md b/metron-analytics/metron-profiler-storm/README.md index 345f38a942..c952cb74dc 100644 --- a/metron-analytics/metron-profiler-storm/README.md +++ b/metron-analytics/metron-profiler-storm/README.md @@ -15,202 +15,27 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# Metron Profiler +# Metron Profiler for Storm -The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. - -This is achieved by summarizing the streaming telemetry data consumed by Metron over sliding windows. A summary statistic is applied to the data received within a given window. Collecting this summary across many windows results in a time series that is useful for analysis. - -Any field contained within a message can be used to generate a profile. A profile can even be produced by combining fields that originate in different data sources. A user has considerable power to transform the data used in a profile by leveraging the Stellar language. A user only need configure the desired profiles and ensure that the Profiler topology is running. +This project allows profiles to be executed using [Apache Storm](https://storm.apache.org). This is a port of the Profiler to Storm that builds low-latency profiles over streaming data sets. +* [Introduction](#introduction) +* [Getting Started](#getting-started) * [Installation](#installation) -* [Creating Profiles](#creating-profiles) -* [Deploying Profiles](#deploying-profiles) -* [Anatomy of a Profile](#anatomy-of-a-profile) * [Configuring the Profiler](#configuring-the-profiler) -* [Examples](#examples) * [Implementation](#implementation) -## Installation - -The Profiler can be installed with either of these two methods. - - * [Ambari Installation](#ambari-installation) - * [Manual Installation](#manual-installation) - -### Ambari Installation - -The Metron Profiler is installed automatically when installing Metron using the Ambari MPack. You can skip the [Installation](#installation) section and move ahead to [Creating Profiles](#creating-profiles) should this be the case. - -### Manual Installation - -This section will describe the steps necessary to manually install the Profiler on an RPM-based Linux distribution. This assumes that core Metron has already been installed and validated. If you installed Metron using the [Ambari MPack](#ambari-mpack), then the Profiler has already been installed and you can skip this section. - -1. Build the Metron RPMs (see Building the [RPMs](../../metron-deployment#rpms)). - - You may have already built the Metron RPMs when core Metron was installed. - - ``` - $ find metron-deployment/ -name "metron-profiler*.rpm" - metron-deployment//packaging/docker/rpm-docker/RPMS/noarch/metron-profiler-0.4.1-201707131420.noarch.rpm - ``` - -1. Copy the Profiler RPM to the installation host. - - The installation host must be the same host on which core Metron was installed. Depending on how you installed Metron, the Profiler RPM might have already been copied to this host with the other Metron RPMs. - - ``` - [root@node1 ~]# find /localrepo/ -name "metron-profiler*.rpm" - /localrepo/metron-profiler-0.4.1-201707112313.noarch.rpm - ``` - -1. Install the RPM. - - ``` - [root@node1 ~]# rpm -ivh metron-profiler-*.noarch.rpm - Preparing... ########################################### [100%] - 1:metron-profiler ########################################### [100%] - ``` - - ``` - [root@node1 ~]# rpm -ql metron-profiler - /usr/metron - /usr/metron/0.4.2 - /usr/metron/0.4.2/bin - /usr/metron/0.4.2/bin/start_profiler_topology.sh - /usr/metron/0.4.2/config - /usr/metron/0.4.2/config/profiler.properties - /usr/metron/0.4.2/flux - /usr/metron/0.4.2/flux/profiler - /usr/metron/0.4.2/flux/profiler/remote.yaml - /usr/metron/0.4.2/lib - /usr/metron/0.4.2/lib/metron-profiler-0.4.2-uber.jar - ``` - -1. Edit the configuration file located at `$METRON_HOME/config/profiler.properties`. - ``` - kafka.zk=node1:2181 - kafka.broker=node1:6667 - ``` - * Change `kafka.zk` to refer to Zookeeper in your environment. - * Change `kafka.broker` to refer to a Kafka Broker in your environment. - -1. Create a table within HBase that will store the profile data. By default, the table is named `profiler` with a column family `P`. The table name and column family must match the Profiler's configuration (see [Configuring the Profiler](#configuring-the-profiler)). - - ``` - $ /usr/hdp/current/hbase-client/bin/hbase shell - hbase(main):001:0> create 'profiler', 'P' - ``` - -1. Start the Profiler topology. - ``` - $ cd $METRON_HOME - $ bin/start_profiler_topology.sh - ``` - -At this point the Profiler is running and consuming telemetry messages. We have not defined any profiles yet, so it is not doing anything very useful. The next section walks you through the steps to create your very first "Hello, World!" profile. - -## Creating Profiles - -This section will describe how to create your very first "Hello, World" profile. It will also outline a useful workflow for creating, testing, and deploying profiles. - -Creating and refining profiles is an iterative process. Iterating against a live stream of data is slow, difficult and error prone. The Profile Debugger was created to provide a controlled and isolated execution environment to create, refine and troubleshoot profiles. - -1. Launch the Stellar Shell. We will leverage the Profiler Debugger from within the Stellar Shell. - ``` - [root@node1 ~]# $METRON_HOME/bin/stellar - Stellar, Go! - [Stellar]>>> %functions PROFILER - PROFILER_APPLY, PROFILER_FLUSH, PROFILER_INIT - ``` - -1. Create a simple `hello-world` profile that will count the number of messages for each `ip_src_addr`. The `SHELL_EDIT` function will open an editor in which you can copy/paste the following Profiler configuration. - ``` - [Stellar]>>> conf := SHELL_EDIT() - [Stellar]>>> conf - { - "profiles": [ - { - "profile": "hello-world", - "onlyif": "exists(ip_src_addr)", - "foreach": "ip_src_addr", - "init": { "count": "0" }, - "update": { "count": "count + 1" }, - "result": "count" - } - ] - } - ``` - -1. Create a Profile execution environment; the Profile Debugger. - - The Profiler will output the number of profiles that have been defined, the number of messages that have been applied and the number of routes that have been followed. - - A route is defined when a message is applied to a specific profile. - * If a message is not needed by any profile, then there are no routes. - * If a message is needed by one profile, then one route has been followed. - * If a message is needed by two profiles, then two routes have been followed. - - ``` - [Stellar]>>> profiler := PROFILER_INIT(conf) - [Stellar]>>> profiler - Profiler{1 profile(s), 0 messages(s), 0 route(s)} - ``` - -1. Create a message that mimics the telemetry that your profile will consume. - - This message can be as simple or complex as you like. For the `hello-world` profile, all you need is a message containing an `ip_src_addr` field. - - ``` - [Stellar]>>> msg := SHELL_EDIT() - [Stellar]>>> msg - { - "ip_src_addr": "10.0.0.1" - } - ``` - -1. Apply the message to your Profiler, as many times as you like. - - ``` - [Stellar]>>> PROFILER_APPLY(msg, profiler) - Profiler{1 profile(s), 1 messages(s), 1 route(s)} - ``` - ``` - [Stellar]>>> PROFILER_APPLY(msg, profiler) - Profiler{1 profile(s), 2 messages(s), 2 route(s)} - ``` - -1. Flush the Profiler. +## Introduction - A flush is what occurs at the end of each 15 minute period in the Profiler. The result is a list of Profile Measurements. Each measurement is a map containing detailed information about the profile data that has been generated. The `value` field is what is written to HBase when running this profile in the Profiler topology. - - There will always be one measurement for each [profile, entity] pair. This profile simply counts the number of messages by IP source address. Notice that the value is '3' for the entity '10.0.0.1' as we applied 3 messages with an 'ip_src_addr' of ’10.0.0.1'. - - ``` - [Stellar]>>> values := PROFILER_FLUSH(profiler) - [Stellar]>>> values - [{period={duration=900000, period=1669628, start=1502665200000, end=1502666100000}, - profile=hello-world, groups=[], value=3, entity=10.0.0.1}] - ``` +The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. -1. Apply real, live telemetry to your profile. +This is achieved by summarizing the streaming telemetry data consumed by Metron over sliding windows. A summary statistic is applied to the data received within a given window. Collecting this summary across many windows results in a time series that is useful for analysis. - Once you are happy with your profile against a controlled data set, it can be useful to introduce more complex, live data. This example extracts 10 messages of live, enriched telemetry to test your profile(s). - ``` - [Stellar]>>> %define bootstrap.servers := "node1:6667" - node1:6667 - [Stellar]>>> msgs := KAFKA_GET("indexing", 10) - [Stellar]>>> LENGTH(msgs) - 10 - ``` - Apply those 10 messages to your profile(s). - ``` - [Stellar]>>> PROFILER_APPLY(msgs, profiler) - Profiler{1 profile(s), 10 messages(s), 10 route(s)} - ``` +Any field contained within a message can be used to generate a profile. A profile can even be produced by combining fields that originate in different data sources. A user has considerable power to transform the data used in a profile by leveraging the Stellar language. A user only need configure the desired profiles and ensure that the Profiler topology is running. +For an introduction to the Profiler, see the [Profiler README](../metron-profiler-common/README.md). -## Deploying Profiles +## Getting Started This section will describe the steps required to get your first "Hello, World!"" profile running. This assumes that you have a successful Profiler [Installation](#installation) and have it running. You can deploy profiles in two different ways. @@ -222,13 +47,13 @@ This section will describe the steps required to get your first "Hello, World!"" Continuing the previous running example, at this point, you have seen how your profile behaves against real, live telemetry in a controlled execution environment. The next step is to deploy your profile to the live, actively running Profiler topology. 1. Start the Stellar Shell with the `-z` command line argument so that a connection to Zookeeper is established. This is required when deploying a new profile definition as shown in the steps below. - ``` - [root@node1 ~]# source /etc/default/metron - [root@node1 ~]# $METRON_HOME/bin/stellar -z $ZOOKEEPER - Stellar, Go! - [Stellar]>>> - [Stellar]>>> %functions CONFIG CONFIG_GET, CONFIG_PUT - ``` + ``` + [root@node1 ~]# source /etc/default/metron + [root@node1 ~]# $METRON_HOME/bin/stellar -z $ZOOKEEPER + Stellar, Go! + [Stellar]>>> + [Stellar]>>> %functions CONFIG CONFIG_GET, CONFIG_PUT + ``` 1. If you haven't already, define your profile. ``` @@ -329,193 +154,84 @@ Continuing the previous running example, at this point, you have seen how your p It is assumed that the `PROFILE_GET` client is correctly configured to match the Profile configuration before using it to read that Profile. More information on configuring and using the Profiler client can be found [here](../metron-profiler-client). -## Anatomy of a Profile - -### Profiler - -The Profiler configuration contains only two fields; only one of which is required. - -``` -{ - "profiles": [ - { "profile": "one", ... }, - { "profile": "two", ... } - ], - "timestampField": "timestamp" -} -``` - -| Name | | Description -|--- |--- |--- -| [profiles](#profiles) | Required | A list of zero or more Profile definitions. -| [timestampField](#timestampfield) | Optional | Indicates whether processing time or event time should be used. By default, processing time is enabled. - - -#### `profiles` - -*Required* - -A list of zero or more Profile definitions. - -#### `timestampField` - -*Optional* - -Indicates whether processing time or event time is used. By default, processing time is enabled. - -##### Processing Time - -By default, no `timestampField` is defined. In this case, the Profiler uses system time when generating profiles. This means that the profiles are generated based on when the data has been processed by the Profiler. This is also known as 'processing time'. - -This is the simplest mode of operation, but has some draw backs. If the Profiler is consuming live data and all is well, the processing and event times will likely remain similar and consistent. If processing time diverges from event time, then the Profiler will generate skewed profiles. - -There are a few scenarios that might cause skewed profiles when using processing time. For example when a system has undergone a scheduled maintenance window and is restarted, a high volume of messages will need to be processed by the Profiler. The output of the Profiler might indicate an increase in activity during this time, although no change in activity actually occurred on the target network. The same situation could occur if an upstream system which provides telemetry undergoes an outage. - -[Event Time](#event-time) can be used to mitigate these problems. - -##### Event Time - -Alternatively, a `timestampField` can be defined. This must be the name of a field contained within the telemetry processed by the Profiler. The Profiler will extract and use the timestamp contained within this field. - -* If a message does not contain this field, it will be dropped. - -* The field must contain a timestamp in epoch milliseconds expressed as either a numeric or string. Otherwise, the message will be dropped. - -* The Profiler will use the same field across all telemetry sources and for all profiles. - -* Be aware of clock skew across telemetry sources. If your profile is processing telemetry from multiple sources where the clock differs significantly, the Profiler may assume that some of those messages are late and will be ignored. Adjusting the [`profiler.window.duration`](#profilerwindowduration) and [`profiler.window.lag`](#profilerwindowlag) can help accommodate skewed clocks. - -### Profiles - -A profile definition requires a JSON-formatted set of elements, many of which can contain Stellar code. The specification contains the following elements. (For the impatient, skip ahead to the [Examples](#examples).) - -| Name | | Description -|--- |--- |--- -| [profile](#profile) | Required | Unique name identifying the profile. -| [foreach](#foreach) | Required | A separate profile is maintained "for each" of these. -| [onlyif](#onlyif) | Optional | Boolean expression that determines if a message should be applied to the profile. -| [groupBy](#groupby) | Optional | One or more Stellar expressions used to group the profile measurements when persisted. -| [init](#init) | Optional | One or more expressions executed at the start of a window period. -| [update](#update) | Required | One or more expressions executed when a message is applied to the profile. -| [result](#result) | Required | Stellar expressions that are executed when the window period expires. -| [expires](#expires) | Optional | Profile data is purged after this period of time, specified in days. - -### `profile` - -*Required* - -A unique name identifying the profile. The field is treated as a string. - -### `foreach` - -*Required* - -A separate profile is maintained 'for each' of these. This is effectively the entity that the profile is describing. The field is expected to contain a Stellar expression whose result is the entity name. - -For example, if `ip_src_addr` then a separate profile would be maintained for each unique IP source address in the data; 10.0.0.1, 10.0.0.2, etc. - -### `onlyif` - -*Optional* - -An expression that determines if a message should be applied to the profile. A Stellar expression that returns a Boolean is expected. A message is only applied to a profile if this expression is true. This allows a profile to filter the messages that get applied to it. - -### `groupBy` - -*Optional* - -One or more Stellar expressions used to group the profile measurements when persisted. This can be used to sort the Profile data to allow for a contiguous scan when accessing subsets of the data. This is also one way to deal with calendar effects. For example, where activity on a weekday can be very different from a weekend. - -A common use case would be grouping by day of week. This allows a contiguous scan to access all profile data for Mondays only. Using the following definition would achieve this. - -``` -"groupBy": [ "DAY_OF_WEEK(start)" ] -``` - -The expression can reference any of these variables. -* Any variable defined by the profile in its `init` or `update` expressions. -* `profile` The name of the profile. -* `entity` The name of the entity being profiled. -* `start` The start time of the profile period in epoch milliseconds. -* `end` The end time of the profile period in epoch milliseconds. -* `duration` The duration of the profile period in milliseconds. -* `result` The result of executing the `result` expression. - -### `init` - -*Optional* +## Installation -One or more expressions executed at the start of a window period. A map is expected where the key is the variable name and the value is a Stellar expression. The map can contain zero or more variable:expression pairs. At the start of each window period, each expression is executed once and stored in the given variable. Note that constant init values such as "0" must be in quotes regardless of their type, as the init value must be a string to be executed by Stellar. +The Profiler can be installed with either of these two methods. -``` -"init": { - "var1": "0", - "var2": "1" -} -``` + * [Ambari Installation](#ambari-installation) + * [Manual Installation](#manual-installation) -### `update` +### Ambari Installation -*Required* +The Metron Profiler is installed automatically when installing Metron using the Ambari MPack. You can skip the [Installation](#installation) section and move ahead to [Creating Profiles](#creating-profiles) should this be the case. -One or more expressions executed when a message is applied to the profile. A map is expected where the key is the variable name and the value is a Stellar expression. The map can include 0 or more variables/expressions. When each message is applied to the profile, the expression is executed and stored in a variable with the given name. +### Manual Installation -``` -"update": { - "var1": "var1 + 1", - "var2": "var2 + 1" -} -``` +This section will describe the steps necessary to manually install the Profiler on an RPM-based Linux distribution. This assumes that core Metron has already been installed and validated. If you installed Metron using the [Ambari MPack](#ambari-mpack), then the Profiler has already been installed and you can skip this section. -### `result` +1. Build the Metron RPMs (see Building the [RPMs](../../metron-deployment#rpms)). -*Required* + You may have already built the Metron RPMs when core Metron was installed. -Stellar expressions that are executed when the window period expires. The expressions are expected to summarize the messages that were applied to the profile over the window period. In the most basic form a single result is persisted for later retrieval. -``` -"result": "var1 + var2" -``` + ``` + $ find metron-deployment/ -name "metron-profiler*.rpm" + metron-deployment//packaging/docker/rpm-docker/RPMS/noarch/metron-profiler-0.4.1-201707131420.noarch.rpm + ``` -For more advanced use cases, a profile can generate two types of results. A profile can define one or both of these result types at the same time. -* `profile`: A required expression that defines a value that is persisted for later retrieval. -* `triage`: An optional expression that defines values that are accessible within the Threat Triage process. +1. Copy the Profiler RPM to the installation host. -**profile** + The installation host must be the same host on which core Metron was installed. Depending on how you installed Metron, the Profiler RPM might have already been copied to this host with the other Metron RPMs. -A required Stellar expression that results in a value that is persisted in the profile store for later retrieval. The expression can result in any object that is Kryo serializable. These values can be retrieved for later use with the [Profiler Client](../metron-profiler-client). -``` -"result": { - "profile": "2 + 2" -} -``` + ``` + [root@node1 ~]# find /localrepo/ -name "metron-profiler*.rpm" + /localrepo/metron-profiler-0.4.1-201707112313.noarch.rpm + ``` -An alternative, simplified form is also acceptable. -``` -"result": "2 + 2" -``` +1. Install the RPM. -**triage** + ``` + [root@node1 ~]# rpm -ivh metron-profiler-*.noarch.rpm + Preparing... ########################################### [100%] + 1:metron-profiler ########################################### [100%] + ``` -An optional map of one or more Stellar expressions. The value of each expression is made available to the Threat Triage process under the given name. Each expression must result in a either a primitive type, like an integer, long, or short, or a String. All other types will result in an error. + ``` + [root@node1 ~]# rpm -ql metron-profiler + /usr/metron + /usr/metron/0.4.2 + /usr/metron/0.4.2/bin + /usr/metron/0.4.2/bin/start_profiler_topology.sh + /usr/metron/0.4.2/config + /usr/metron/0.4.2/config/profiler.properties + /usr/metron/0.4.2/flux + /usr/metron/0.4.2/flux/profiler + /usr/metron/0.4.2/flux/profiler/remote.yaml + /usr/metron/0.4.2/lib + /usr/metron/0.4.2/lib/metron-profiler-0.4.2-uber.jar + ``` -In the following example, three values, the minimum, the maximum and the mean are appended to a message. This message is consumed by Metron, like other sources of telemetry, and each of these values are accessible from within the Threat Triage process using the given field names; `min`, `max`, and `mean`. -``` -"result": { - "triage": { - "min": "STATS_MIN(stats)", - "max": "STATS_MAX(stats)", - "mean": "STATS_MEAN(stats)" - } -} -``` +1. Edit the configuration file located at `$METRON_HOME/config/profiler.properties`. + ``` + kafka.zk=node1:2181 + kafka.broker=node1:6667 + ``` + * Change `kafka.zk` to refer to Zookeeper in your environment. + * Change `kafka.broker` to refer to a Kafka Broker in your environment. -### `expires` +1. Create a table within HBase that will store the profile data. By default, the table is named `profiler` with a column family `P`. The table name and column family must match the Profiler's configuration (see [Configuring the Profiler](#configuring-the-profiler)). -*Optional* + ``` + $ /usr/hdp/current/hbase-client/bin/hbase shell + hbase(main):001:0> create 'profiler', 'P' + ``` -A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire. +1. Start the Profiler topology. + ``` + $ cd $METRON_HOME + $ bin/start_profiler_topology.sh + ``` -The REPL can be a powerful for developing profiles. Read all about [Developing Profiles](../metron-profiler-client/#developing_profiles). +At this point the Profiler is running and consuming telemetry messages. We have not defined any profiles yet, so it is not doing anything very useful. The next section walks you through the steps to create your very first "Hello, World!" profile. ## Configuring the Profiler @@ -682,217 +398,3 @@ For these classes, Storm will uses Kryo's `FieldSerializer` as defined in the [S This value should only need altered if you have defined a profile that results in a non-primitive, user-defined type that is not in this list. If the class is not defined in this list, Java serialization will be used and the class must adhere to Java's serialization requirements. The performance of the entire Profiler topology can be negatively impacted if any profile produces results that undergo Java serialization. - -## Examples - -The following examples are intended to highlight the functionality provided by the Profiler. Try out these examples easily in the Stellar Shell as described in the [Creating Profiles](#creating-profiles) section. - -These examples assume a fictitious input message stream that looks like the following. -``` -[Stellar]>>> msgs := SHELL_EDIT() -[Stellar]>>> msgs -[ - { - "ip_src_addr": "10.0.0.1", - "protocol": "HTTPS", - "length": "10", - "bytes_in": "234" - }, - { - "ip_src_addr": "10.0.0.2", - "protocol": "HTTP", - "length": "20", - "bytes_in": "390" - }, - { - "ip_src_addr": "10.0.0.3", - "protocol": "DNS", - "length": "30", - "bytes_in": "560" - } -] -``` - - -### Example 1 - -The total number of bytes of HTTP data for each host. The following configuration would be used to generate this profile. - -``` -{ - "profiles": [ - { - "profile": "example1", - "foreach": "ip_src_addr", - "onlyif": "protocol == 'HTTP'", - "init": { - "total_bytes": 0.0 - }, - "update": { - "total_bytes": "total_bytes + bytes_in" - }, - "result": "total_bytes", - "expires": 30 - } - ] -} -``` - -This creates a profile... - * Named ‘example1’ - * That for each IP source address - * Only if the 'protocol' field equals 'HTTP' - * Initializes a counter ‘total_bytes’ to zero - * Adds to ‘total_bytes’ the value of the message's ‘bytes_in’ field - * Returns ‘total_bytes’ as the result - * The profile data will expire in 30 days - -### Example 2 - -The ratio of DNS traffic to HTTP traffic for each host. The following configuration would be used to generate this profile. - -``` -{ - "profiles": [ - { - "profile": "example2", - "foreach": "ip_src_addr", - "onlyif": "protocol == 'DNS' or protocol == 'HTTP'", - "init": { - "num_dns": 1.0, - "num_http": 1.0 - }, - "update": { - "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)", - "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)" - }, - "result": "num_dns / num_http" - } - ] -} -``` - -This creates a profile... - * Named ‘example2’ - * That for each IP source address - * Only if the 'protocol' field equals 'HTTP' or 'DNS' - * Accumulates the number of DNS requests - * Accumulates the number of HTTP requests - * Returns the ratio of these as the result - -### Example 3 - -The average of the `length` field of HTTP traffic. The following configuration would be used to generate this profile. - -``` -{ - "profiles": [ - { - "profile": "example3", - "foreach": "ip_src_addr", - "onlyif": "protocol == 'HTTP'", - "update": { "s": "STATS_ADD(s, length)" }, - "result": "STATS_MEAN(s)" - } - ] -} -``` - -This creates a profile... - * Named ‘example3’ - * That for each IP source address - * Only if the 'protocol' field is 'HTTP' - * Adds the `length` field from each message - * Calculates the average as the result - -### Example 4 - -It is important to note that the Profiler can persist any serializable Object, not just numeric values. An alternative to the previous example could take advantage of this. - -Instead of storing the mean of the lengths, the profile could store a statistical summarization of the lengths. This summary can then be used at a later time to calculate the mean, min, max, percentiles, or any other sensible metric. This provides a much greater degree of flexibility. - -``` -{ - "profiles": [ - { - "profile": "example4", - "foreach": "ip_src_addr", - "onlyif": "protocol == 'HTTP'", - "update": { "s": "STATS_ADD(s, length)" }, - "result": "s" - } - ] -} -``` - -The following Stellar REPL session shows how you might use this summary to calculate different metrics with the same underlying profile data. -It is assumed that the PROFILE_GET client is configured as described [here](../metron-profiler-client). - -Retrieve the last 30 minutes of profile measurements for a specific host. -``` -$ source /etc/default/metron -$ bin/stellar -z $ZOOKEEPER - -[Stellar]>>> stats := PROFILE_GET( "example4", "10.0.0.1", PROFILE_FIXED(30, "MINUTES")) -[Stellar]>>> stats -[org.apache.metron.common.math.stats.OnlineStatisticsProvider@79fe4ab9, ...] -``` - -Calculate different metrics with the same profile data. -``` -[Stellar]>>> STATS_MEAN( GET_FIRST( stats)) -15979.0625 - -[Stellar]>>> STATS_PERCENTILE( GET_FIRST(stats), 90) -30310.958 -``` - -Merge all of the profile measurements over the past 30 minutes into a single summary and calculate the 90th percentile. -``` -[Stellar]>>> merged := STATS_MERGE( stats) -[Stellar]>>> STATS_PERCENTILE(merged, 90) -29810.992 -``` - -More information on accessing profile data can be found in the [Profiler Client](../metron-profiler-client). - -More information on using the [`STATS_*` functions in Stellar can be found here](../../metron-platform/metron-common). - -### `profiler.writer.batchSize` - -*Default*: 15 - -The number of records to batch when writing to Kakfa. This is managed in the global configuration and does not require a topology restart. - -### `profiler.writer.batchTimeout` - -*Default*: 0 - -The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to `0`, it defaults to a system-determined duration which is a fraction of the Storm parameter `topology.message.timeout.secs`. -Ignored if batchSize is `1`, since this disables batching. This is managed in the global configuration and does not require a topology restart. - -## Implementation - -## Key Classes - -* `ProfileMeasurement` - Represents a single data point within a Profile. A Profile is effectively a time series. To this end a Profile is composed of many ProfileMeasurement values which in aggregate form a time series. - -* `ProfilePeriod` - The Profiler captures one `ProfileMeasurement` each `ProfilePeriod`. A `ProfilePeriod` will occur at fixed, deterministic points in time. This allows for efficient retrieval of profile data. - -* `RowKeyBuilder` - Builds row keys that can be used to read or write profile data to HBase. - -* `ColumnBuilder` - Defines the columns of data stored with a profile measurement. - -* `ProfileHBaseMapper` - Defines for the `HBaseBolt` how profile measurements are stored in HBase. This class leverages a `RowKeyBuilder` and `ColumnBuilder`. - -## Storm Topology - -The Profiler is implemented as a Storm topology using the following bolts and spouts. - -* `KafkaSpout` - A spout that consumes messages from a single Kafka topic. In most cases, the Profiler topology will consume messages from the `indexing` topic. This topic contains fully enriched messages that are ready to be indexed. This ensures that profiles can take advantage of all the available data elements. - -* `ProfileSplitterBolt` - The bolt responsible for filtering incoming messages and directing each to the one or more downstream bolts that are responsible for building a profile. Each message may be needed by 0, 1 or even many profiles. Each emitted tuple contains the 'resolved' entity name, the profile definition, and the input message. - -* `ProfileBuilderBolt` - This bolt maintains all of the state required to build a profile. When the window period expires, the data is summarized as a `ProfileMeasurement`, all state is flushed, and the `ProfileMeasurement` is emitted. Each instance of this bolt is responsible for maintaining the state for a single Profile-Entity pair. - -* `HBaseBolt` - A bolt that is responsible for writing to HBase. Most profiles will be flushed every 15 minutes or so. If each `ProfileBuilderBolt` were responsible for writing to HBase itself, there would be little to no opportunity to optimize these writes. By aggregating the writes from multiple Profile-Entity pairs these writes can be batched, for example. From d9d0bbf4cb5d34dfb0e5ca5b9b01d22fbdf0634d Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Mon, 10 Sep 2018 14:08:33 -0400 Subject: [PATCH 4/6] Fixed grammar in README. Thanks @merrimanr --- metron-analytics/metron-profiler-common/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-analytics/metron-profiler-common/README.md b/metron-analytics/metron-profiler-common/README.md index 1f60014c6f..8f26aaf9a7 100644 --- a/metron-analytics/metron-profiler-common/README.md +++ b/metron-analytics/metron-profiler-common/README.md @@ -43,13 +43,13 @@ There are three separate ports of the Profiler that share this common code base. 1. [Deploy your profile](../metron-profiler-storm/README.md#getting-started) to Storm to maintain a low-latency profile over a streaming data set. -1. [Retrieve your profile data](../metron-profiler-client/README.md) using the Stellar API so that you can build enrichments, alert on abnormalities +1. [Retrieve your profile data](../metron-profiler-client/README.md) using the Stellar API so that you can build enrichments, alert on abnormalities. 1. Explore more ways to create [profiles](#more-examples). ## Profiles -Let's start with a simple example. The following profile maintains a count of the number of telemetry messages for each IP source address. A counter is initialized to 0, then incremented each time a message is received for a give IP source address. At regular intervals the count is flushed and stored. Over time this results in a time series describing the amount of telemetry received for each IP source address. +Let's start with a simple example. The following profile maintains a count of the number of telemetry messages for each IP source address. A counter is initialized to 0, then incremented each time a message is received for a given IP source address. At regular intervals the count is flushed and stored. Over time this results in a time series describing the amount of telemetry received for each IP source address. ``` { From bc2b9392e63b484fbcf2d803d00b3766a6ff8f7c Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Mon, 10 Sep 2018 14:19:17 -0400 Subject: [PATCH 5/6] Moved the integration test under the new package structure --- .../apache/metron/profiler/storm/ProfileBuilderBoltTest.java | 2 +- .../{ => storm}/integration/ConfigUploadComponent.java | 4 ++-- .../profiler/{ => storm}/integration/MessageBuilder.java | 2 +- .../{ => storm}/integration/ProfilerIntegrationTest.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/{ => storm}/integration/ConfigUploadComponent.java (98%) rename metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/{ => storm}/integration/MessageBuilder.java (97%) rename metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/{ => storm}/integration/ProfilerIntegrationTest.java (99%) diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java index 44556646d1..fc94afa498 100644 --- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java +++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java @@ -25,7 +25,7 @@ import org.apache.metron.profiler.MessageDistributor; import org.apache.metron.profiler.MessageRoute; import org.apache.metron.profiler.ProfileMeasurement; -import org.apache.metron.profiler.integration.MessageBuilder; +import org.apache.metron.profiler.storm.integration.MessageBuilder; import org.apache.metron.test.bolt.BaseBoltTest; import org.apache.storm.task.OutputCollector; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java similarity index 98% rename from metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java index b59d0b5e30..70487a084f 100644 --- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java +++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java @@ -17,7 +17,7 @@ * limitations under the License. * */ -package org.apache.metron.profiler.integration; +package org.apache.metron.profiler.storm.integration; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -121,4 +121,4 @@ public ConfigUploadComponent withProfilerConfiguration(String path) { this.profilerConfiguration = path; return this; } -} \ No newline at end of file +} diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/MessageBuilder.java similarity index 97% rename from metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/MessageBuilder.java index 7e1628ba85..17e36e19f2 100644 --- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/MessageBuilder.java +++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/MessageBuilder.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.integration; +package org.apache.metron.profiler.storm.integration; import org.json.simple.JSONObject; diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java similarity index 99% rename from metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java rename to metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java index 8f7741d352..182600ab61 100644 --- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.profiler.integration; +package org.apache.metron.profiler.storm.integration; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.hbase.Cell; From 3684d61ffc2809455f0ed439fb6ee84ed2a80148 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Mon, 10 Sep 2018 14:19:59 -0400 Subject: [PATCH 6/6] Rename the artifact ID --- metron-analytics/metron-profiler-storm/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-analytics/metron-profiler-storm/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml index ea58021333..22c62551af 100644 --- a/metron-analytics/metron-profiler-storm/pom.xml +++ b/metron-analytics/metron-profiler-storm/pom.xml @@ -20,7 +20,7 @@ metron-analytics 0.5.1 - metron-profiler + metron-profiler-storm https://metron.apache.org/ UTF-8