From 9cfa0181a95d0b2b21bd665530119e7b261962dc Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 7 Jun 2017 16:56:04 -0500 Subject: [PATCH 1/9] initial commit --- .../CURRENT/configuration/metron-env.xml | 340 +++++++-- .../package/scripts/indexing_master.py | 9 + .../CURRENT/package/scripts/metron_service.py | 11 +- .../package/scripts/params/params_linux.py | 40 + .../templates/elasticsearch.properties.j2 | 49 ++ .../templates/enrichment.properties.j2 | 80 +- .../CURRENT/package/templates/global.json.j2 | 6 + .../METRON/CURRENT/themes/metron_theme.json | 688 ++++++++++++++++-- .../src/main/config/elasticsearch.properties | 52 +- .../src/main/config/enrichment.properties | 76 +- .../src/main/flux/enrichment/remote.yaml | 32 +- .../src/main/flux/indexing/remote.yaml | 16 +- 12 files changed, 1122 insertions(+), 277 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 3e5a405f6c..0e0efda1f0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -253,69 +253,295 @@ - global-json - global.json template - This is the jinja template for global.json file - -{ -"es.clustername": "{{ es_cluster_name }}", -"es.ip": "{{ es_url }}", -"es.date.format": "yyyy.MM.dd.HH", -"parser.error.topic": "indexing" -} - + es_date_format + Elasticsearch Date Format + yyyy.MM.dd.HH + Elasticsearch Date Format + + + parser_error_topic + Parser Error Topic + indexing + Parser Error Topic + + + enrichment_workers + Number of Enrichment Topology Workers + 1 + Enrichment Workers + + + enrichment_acker_executors + Number of Enrichment Topology Ackers + 1 + Enrichment Ackers + + + enrichment_topology_worker_childopts + Enrichment Topology JVM Options + + Enrichment childopts - content + true - elasticsearch-properties - The template for the elasticsearch.properties file. - elasticsearch.properties template - -##### Storm ##### -indexing.workers=1 -indexing.executors=0 -topology.worker.childopts={{topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -##### Kafka ##### -kafka.zk={{ zookeeper_quorum }} -kafka.broker={{ kafka_brokers }} -kafka.start=UNCOMMITTED_EARLIEST -kafka.security.protocol={{kafka_security_protocol}} -storm.auto.credentials=[] -##### Indexing ##### -index.input.topic=indexing -index.error.topic=indexing -writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter -##### Metrics ##### -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false -#Graphite Addresses -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true -##### HDFS ##### -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy -bolt.hdfs.rotation.policy.units=DAYS -bolt.hdfs.rotation.policy.count=1 -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url={{ default_fs }} -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec -index.hdfs.output={{ metron_apps_indexed_hdfs_dir }} - + enrichment_topology_max_spout_pending + Enrichment Topology Spout Max Pending Tuples + + Enrichment Max Pending + + true + + + + enrichment_kafka_start + Enrichment Topology Spout Offset + UNCOMMITTED_EARLIEST + Enrichment Offset + + value-list + + + EARLIEST + + + LATEST + + + UNCOMMITTED_EARLIEST + + + UNCOMMITTED_LATEST + + + 1 + + + + enrichment_input_topic + Enrichment Input Topic + enrichments + Enrichment Input Topic + + + enrichment_output_topic + Enrichment Output Topic + indexing + Enrichment Output Topic + + + enrichment_error_topic + Enrichment Error Topic + indexing + Enrichment Error Topic + + + threatintel_error_topic + Threat Intel Error Topic + indexing + Threat Intel Error Topic + + + enrichment_join_cache_size + Enrichment Join Bolt Cache Max Size + 100000 + Enrichment Join Size + + + threatintel_join_cache_size + Threat Intel Join Bolt Cache Max Size + 100000 + Threat Intel Join Size + + + enrichment_host_known_hosts + List of Known Hosts for Host Enrichment + [{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + Host Enrichment content - + + + enrichment_kafka_spout_parallelism + Enrichment Topology Kafka Spout Parallelism + 1 + Enrichment Spout Parallelism + + + enrichment_split_parallelism + Enrichment Topology Enrichment Split Bolt Parallelism + 1 + Enrichment Split Parallelism + + + enrichment_stellar_parallelism + Enrichment Topology Enrichment Stellar Bolt Parallelism + 1 + Enrichment Stellar Parallelism + + + enrichment_join_parallelism + Enrichment Topology Enrichment Join Bolt Parallelism + 1 + Enrichment Join Parallelism + + + threat_intel_split_parallelism + Enrichment Topology Threat Intel Split Bolt Parallelism + 1 + Threat Intel Spout Parallelism + + + threat_intel_stellar_parallelism + Enrichment Topology Threat Intel Stellar Bolt Parallelism + 1 + Threat Intel Spout Parallelism + + + threat_intel_join_parallelism + Enrichment Topology Threat Intel Join Bolt Parallelism + 1 + Threat Intel Join Parallelism + + + kafka_writer_parallelism + Enrichment Topology Kafka Writer Parallelism + 1 + Enrichment Kafka Writer Parallelism + + + + indexing_workers + Number of Indexing Topology Workers + 1 + Indexing Workers + + + indexing_acker_executors + Number of Indexing Topology Ackers + 1 + Enrichment Ackers + + + indexing_topology_worker_childopts + Indexing Topology JVM Options + + Indexing childopts + + true + + + + indexing_topology_max_spout_pending + Indexing Topology Spout Max Pending Tuples + + Indexing Max Pending + + true + + + + indexing_kafka_start + Indexing Topology Spout Offset + UNCOMMITTED_EARLIEST + Indexing Offset + + value-list + + + EARLIEST + + + LATEST + + + UNCOMMITTED_EARLIEST + + + UNCOMMITTED_LATEST + + + 1 + + + + indexing_input_topic + Indexing Input Topic + indexing + Indexing Input Topic + + + indexing_error_topic + Indexing Error Topic + indexing + Indexing Error Topic + + + indexing_writer_class_name + Indexing Writer Class Name + org.apache.metron.elasticsearch.writer.ElasticsearchWriter + Indexing Writer Class Name + + + bolt_hdfs_rotation_policy + Class Name of Hdfs Rotation Policy + org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy + Hdfs Rotation Policy + + + bolt_hdfs_rotation_policy_units + Frequency of Hdfs File Rotation + DAYS + Hdfs Rotation Policy Units + + value-list + + + SECONDS + + + MINUTES + + + HOURS + + + DAYS + + + 1 + + + + bolt_hdfs_rotation_policy_count + Hdfs Rotation Policy Count + 1 + Hdfs Rotation Policy Count + + + indexing_error_topic + Indexing Error Topic + indexing + Indexing Error Topic + + + indexing_kafka_spout_parallelism + Indexing Topology Kafka Spout Parallelism + 1 + Indexing Spout Parallelism + + + indexing_writer_parallelism + Indexing Topology Writer Bolt Parallelism + 1 + Indexing Writer Parallelism + + + hdfs_writer_parallelism + Indexing Topology Hdfs Bolt Parallelism + 1 + Indexing Hdfs Parallelism diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index f208f3af4f..76b6b60791 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -19,6 +19,8 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.core.resources.system import File +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format from resource_management.core.source import StaticFile from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.script import Script @@ -40,6 +42,13 @@ def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) + Logger.info("Running indexing configure") + File(format("{metron_config_path}/elasticsearch.properties"), + content=Template("elasticsearch.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) + commands = IndexingCommands(params) metron_service.load_global_config(params) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index be1654f260..c0b34f127c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -20,7 +20,7 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Directory, File from resource_management.core.resources.system import Execute -from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.functions.get_user_call_output import get_user_call_output from metron_security import kinit @@ -80,15 +80,12 @@ def load_global_config(params): group=params.metron_group ) - File("{0}/global.json".format(params.metron_zookeeper_config_path), + File(ambari_format("{metron_zookeeper_config_path}/global.json"), + content=Template("global.json.j2"), owner=params.metron_user, - content=InlineTemplate(params.global_json_template) + group=params.metron_group ) - File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'), - owner=params.metron_user, - content=InlineTemplate(params.global_properties_template)) - init_config() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index d4733252ba..86ce519c44 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,6 +39,7 @@ hostname = config['hostname'] metron_home = status_params.metron_home parsers = status_params.parsers +parser_error_topic = config['configurations']['metron-env']['parser_error_topic'] geoip_url = config['configurations']['metron-env']['geoip_url'] geoip_hdfs_dir = "/apps/metron/geo/default/" metron_indexing_topology = status_params.metron_indexing_topology @@ -84,6 +85,7 @@ es_url = ",".join([host + ":" + es_binary_port for host in es_host_list]) es_http_port = config['configurations']['metron-env']['es_http_port'] es_http_url = es_host_list[0] + ":" + es_http_port +es_date_format = config['configurations']['metron-env']['es_date_format'] # hadoop params stack_root = Script.get_stack_root() @@ -223,3 +225,41 @@ # Management UI metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", ['localhost'])[0] + +# Enrichment +enrichment_workers = config['configurations']['metron-env']['enrichment_workers'] +enrichment_acker_executors = config['configurations']['metron-env']['enrichment_acker_executors'] +enrichment_topology_worker_childopts = config['configurations']['metron-env']['enrichment_topology_worker_childopts'] +enrichment_topology_max_spout_pending = config['configurations']['metron-env']['enrichment_topology_max_spout_pending'] +enrichment_kafka_start = config['configurations']['metron-env']['enrichment_kafka_start'] +enrichment_input_topic = config['configurations']['metron-env']['enrichment_input_topic'] +enrichment_output_topic = config['configurations']['metron-env']['enrichment_output_topic'] +enrichment_error_topic = config['configurations']['metron-env']['enrichment_error_topic'] +threatintel_error_topic = config['configurations']['metron-env']['threatintel_error_topic'] +enrichment_join_cache_size = config['configurations']['metron-env']['enrichment_join_cache_size'] +threatintel_join_cache_size = config['configurations']['metron-env']['threatintel_join_cache_size'] +enrichment_host_known_hosts = config['configurations']['metron-env']['enrichment_host_known_hosts'] +enrichment_kafka_spout_parallelism = config['configurations']['metron-env']['enrichment_kafka_spout_parallelism'] +enrichment_split_parallelism = config['configurations']['metron-env']['enrichment_split_parallelism'] +enrichment_stellar_parallelism = config['configurations']['metron-env']['enrichment_stellar_parallelism'] +enrichment_join_parallelism = config['configurations']['metron-env']['enrichment_join_parallelism'] +threat_intel_split_parallelism = config['configurations']['metron-env']['threat_intel_split_parallelism'] +threat_intel_stellar_parallelism = config['configurations']['metron-env']['threat_intel_stellar_parallelism'] +threat_intel_join_parallelism = config['configurations']['metron-env']['threat_intel_join_parallelism'] +kafka_writer_parallelism = config['configurations']['metron-env']['kafka_writer_parallelism'] + +# Threat Intel +indexing_workers = config['configurations']['metron-env']['indexing_workers'] +indexing_acker_executors = config['configurations']['metron-env']['indexing_acker_executors'] +indexing_topology_worker_childopts = config['configurations']['metron-env']['indexing_topology_worker_childopts'] +indexing_topology_max_spout_pending = config['configurations']['metron-env']['indexing_topology_max_spout_pending'] +indexing_kafka_start = config['configurations']['metron-env']['indexing_kafka_start'] +indexing_input_topic = config['configurations']['metron-env']['indexing_input_topic'] +indexing_error_topic = config['configurations']['metron-env']['indexing_error_topic'] +indexing_writer_class_name = config['configurations']['metron-env']['indexing_writer_class_name'] +bolt_hdfs_rotation_policy = config['configurations']['metron-env']['bolt_hdfs_rotation_policy'] +bolt_hdfs_rotation_policy_units = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_units'] +bolt_hdfs_rotation_policy_count = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_count'] +indexing_kafka_spout_parallelism = config['configurations']['metron-env']['indexing_kafka_spout_parallelism'] +indexing_writer_parallelism = config['configurations']['metron-env']['indexing_writer_parallelism'] +hdfs_writer_parallelism = config['configurations']['metron-env']['hdfs_writer_parallelism'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 new file mode 100644 index 0000000000..acb0f59727 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 @@ -0,0 +1,49 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +indexing.workers={{indexing_workers}} +indexing.acker.executors={{indexing_acker_executors}} +topology.worker.childopts={{indexing_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{indexing_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{indexing_kafka_start}} + +indexing.input.topic={{indexing_input_topic}} +indexing.error.topic={{indexing_error_topic}} + +##### Indexing ##### +indexing.writer.class.name={{indexing_writer_class_name}} + +##### HDFS ##### +bolt.hdfs.rotation.policy={{bolt_hdfs_rotation_policy}} +bolt.hdfs.rotation.policy.units={{bolt_hdfs_rotation_policy_units}} +bolt.hdfs.rotation.policy.count={{bolt_hdfs_rotation_policy_count}} +indexing.hdfs.output={{metron_apps_indexed_hdfs_dir}} + +##### Parallelism ##### +kafka.spout.parallelism={{indexing_kafka_spout_parallelism}} +indexing.writer.parallelism={{indexing_writer_parallelism}} +hdfs.writer.parallelism={{hdfs_writer_parallelism}} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 index bdafb451c6..2aca4059d0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 @@ -16,72 +16,48 @@ # limitations under the License. #} -##### Kafka ##### +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} +##### Kafka ##### kafka.zk={{zookeeper_quorum}} kafka.broker={{kafka_brokers}} -kafka.security.protocol={{kafka_security_protocol}} -topology.worker.childopts={{topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -enrichment.output.topic=indexing -enrichment.error.topic=indexing -threat.intel.error.topic=indexing - -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses +kafka.security.protocol=PLAINTEXT -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{enrichment_kafka_start}} -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} +##### JoinBolt ##### +enrichment.join.cache.size={{enrichment_join_cache_size}} +threat.intel.join.cache.size={{threatintel_join_cache_size}} -#GenericEnrichmentBolt -org.apache.metron.metrics.GenericEnrichmentBolt.acks=true -org.apache.metron.metrics.GenericEnrichmentBolt.emits=true -org.apache.metron.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -org.apache.metron.metrics.TelemetryIndexingBolt.acks=true -org.apache.metron.metrics.TelemetryIndexingBolt.emits=true -org.apache.metron.metrics.TelemetryIndexingBolt.fails=true - -##### Host Enrichment ##### - +##### Enrichment ##### hbase.provider.impl=org.apache.metron.hbase.HTableProvider enrichment.simple.hbase.table={{enrichment_table}} enrichment.simple.hbase.cf={{enrichment_cf}} -org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ -{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ -{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] - - -##### HBase ##### -bolt.hbase.table.name=pcap -bolt.hbase.table.fields=t:value -bolt.hbase.table.key.tuple.field.name=key -bolt.hbase.table.timestamp.tuple.field.name=timestamp -bolt.hbase.enable.batching=false -bolt.hbase.write.buffer.size.in.bytes=2000000 -bolt.hbase.durability=SKIP_WAL -bolt.hbase.partitioner.region.info.refresh.interval.mins=60 +enrichment.host.known_hosts={{enrichment_host_known_hosts}} ##### Threat Intel ##### - threat.intel.tracker.table={{threatintel_table}} threat.intel.tracker.cf={{threatintel_cf}} threat.intel.simple.hbase.table={{threatintel_table}} threat.intel.simple.hbase.cf={{threatintel_cf}} -threat.intel.ip.table= -threat.intel.ip.cf= +##### Parallelism ##### +kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} +enrichment.split.parallelism={{enrichment_split_parallelism}} +enrichment.stellar.parallelism={{enrichment_stellar_parallelism}} +enrichment.join.parallelism={{enrichment_join_parallelism}} +threat.intel.split.parallelism={{threat_intel_split_parallelism}} +threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}} +threat.intel.join.parallelism={{threat_intel_join_parallelism}} +kafka.writer.parallelism={{kafka_writer_parallelism}} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 new file mode 100644 index 0000000000..61e1416776 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 @@ -0,0 +1,6 @@ +{ + "es.clustername": "{{ es_cluster_name }}", + "es.ip": "{{ es_url }}", + "es.date.format": "{{es_date_format}}", + "parser.error.topic": "{{parser_error_topic}}" +} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 20aca4696a..15388e7395 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -1,71 +1,639 @@ { "configuration": { - "layouts": [{ - "name": "default", - "tabs": [{ - "name": "metron_general", - "display-name": "Index Settings", - "layout": { - "tab-columns": "1", - "tab-rows": "1", - "sections": [{ - "name": "section-general", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1", - "section-columns": "1", - "section-rows": "1", - "subsections": [{ - "name": "subsection-general-indexing", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }] - }] - } - }] - }], + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "index_settings", + "display-name": "Index Settings", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-index-settings", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-index-settings", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "parsers", + "display-name": "Parsers", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-parsers", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-parsers", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "enrichment", + "display-name": "Enrichment", + "layout": { + "tab-columns": "1", + "tab-rows": "4", + "sections": [ + { + "name": "section-enrichment-adapters", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-adapters", + "display-name": "Adapters", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-enrichment-kafka", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-enrichment-storm", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "indexing", + "display-name": "Indexing", + "layout": { + "tab-columns": "1", + "tab-rows": "3", + "sections": [ + { + "name": "section-indexing-kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-indexing-storm", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-indexing-hdfs", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-hdfs", + "display-name": "Hdfs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], "placement": { "configuration-layout": "default", - "configs": [{ - "config": "metron-env/metron_enrichment_db_port", - "subsection-name": "subsection-general-database" - }, { + "configs": [ + { + "config": "metron-env/es_hosts", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_binary_port", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_http_port", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_cluster_name", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_date_format", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/parsers", + "subsection-name": "subsection-parsers" + }, + { + "config": "metron-env/parser_error_topic", + "subsection-name": "subsection-parsers" + }, + { + "config": "metron-env/geoip_url", + "subsection-name": "subsection-enrichment-adapters" + }, + { + "config": "metron-env/enrichment_host_known_hosts", + "subsection-name": "subsection-enrichment-adapters" + }, + { + "config": "metron-env/enrichment_kafka_start", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_input_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_output_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_error_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/threatintel_error_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_workers", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_acker_executors", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_topology_worker_childopts", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_topology_max_spout_pending", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_join_cache_size", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threatintel_join_cache_size", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_kafka_spout_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_split_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_stellar_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_join_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_split_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_stellar_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_join_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/kafka_writer_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + + { + "config": "metron-env/indexing_kafka_start", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_input_topic", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_error_topic", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_workers", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_acker_executors", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_topology_worker_childopts", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_topology_max_spout_pending", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_kafka_spout_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_writer_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/hdfs_writer_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/metron_apps_indexed_hdfs_dir", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_units", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_count", + "subsection-name": "subsection-indexing-hdfs" + } + ] + }, + "widgets": [ + { "config": "metron-env/es_hosts", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_binary_port", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_http_port", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_cluster_name", - "subsection-name": "subsection-general-indexing" - }] - }, - "widgets": [{ - "config": "metron-env/es_hosts", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_binary_port", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_http_port", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_cluster_name", - "widget": { - "type": "text-field" + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/es_date_format", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/parsers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/parser_error_topic", + "widget": { + "type": "text-field" + } + }, + + + { + "config": "metron-env/enrichment_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/geoip_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_acker_executors", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_topology_max_spout_pending", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/enrichment_input_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_output_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threatintel_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_join_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threatintel_join_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_host_known_hosts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_kafka_spout_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_split_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_stellar_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_join_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_split_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_stellar_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_join_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/kafka_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + + { + "config": "metron-env/indexing_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_acker_executors", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_topology_max_spout_pending", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/indexing_input_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_count", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_kafka_spout_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/hdfs_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + + { + "config": "metron-env/metron_apps_indexed_hdfs_dir", + "widget": { + "type": "text-field" + } } - }] + ] } } diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties index d5efcb2d22..1620dfd4fb 100644 --- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties +++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties @@ -16,58 +16,32 @@ ##### Storm ##### indexing.workers=1 -indexing.executors=0 +indexing.acker.executors=0 topology.worker.childopts= topology.auto-credentials=[''] +topology.max.spout.pending= ##### Kafka ##### - kafka.zk=node1:2181 kafka.broker=node1:6667 -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start=UNCOMMITTED_EARLIEST kafka.security.protocol=PLAINTEXT -##### Indexing ##### -index.input.topic=indexing -index.error.topic=indexing -writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter - -##### ElasticSearch ##### - -es.ip=10.22.0.214 -es.port=9300 -es.clustername=elasticsearch - -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses - -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true +indexing.input.topic=indexing +indexing.error.topic=indexing +##### Indexing ##### +indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter ##### HDFS ##### - -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy bolt.hdfs.rotation.policy.units=DAYS bolt.hdfs.rotation.policy.count=1 +indexing.hdfs.output=/tmp/metron/enriched -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020 -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec -index.hdfs.output=/tmp/metron/enriched +##### Parallelism ##### +kafka.spout.parallelism=1 +indexing.writer.parallelism=1 +hdfs.writer.parallelism=1 diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties index e293201f70..95929683ab 100644 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties +++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties @@ -14,71 +14,51 @@ # See the License for the specific language governing permissions and # limitations under the License. +##### Storm ##### +enrichment.workers=1 +enrichment.acker.executors=0 +topology.worker.childopts= +topology.auto-credentials= +topology.max.spout.pending= ##### Kafka ##### - kafka.zk=node1:2181 kafka.broker=node1:6667 kafka.security.protocol=PLAINTEXT -topology.worker.childopts= +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST + +enrichment.input.topic=enrichments enrichment.output.topic=indexing enrichment.error.topic=indexing threat.intel.error.topic=indexing -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses - -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 - -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -org.apache.metron.metrics.GenericEnrichmentBolt.acks=true -org.apache.metron.metrics.GenericEnrichmentBolt.emits=true -org.apache.metron.metrics.GenericEnrichmentBolt.fails=true +##### JoinBolt ##### +enrichment.join.cache.size=100000 +threat.intel.join.cache.size=100000 - -#TelemetryIndexingBolt -org.apache.metron.metrics.TelemetryIndexingBolt.acks=true -org.apache.metron.metrics.TelemetryIndexingBolt.emits=true -org.apache.metron.metrics.TelemetryIndexingBolt.fails=true - -##### Host Enrichment ##### - -org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +##### Enrichment ##### +hbase.provider.impl=org.apache.metron.hbase.HTableProvider +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t +enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] -##### HBase ##### -bolt.hbase.table.name=pcap -bolt.hbase.table.fields=t:value -bolt.hbase.table.key.tuple.field.name=key -bolt.hbase.table.timestamp.tuple.field.name=timestamp -bolt.hbase.enable.batching=false -bolt.hbase.write.buffer.size.in.bytes=2000000 -bolt.hbase.durability=SKIP_WAL -bolt.hbase.partitioner.region.info.refresh.interval.mins=60 - ##### Threat Intel ##### - threat.intel.tracker.table=access_tracker threat.intel.tracker.cf=t -threat.intel.ip.table= -threat.intel.ip.cf= threat.intel.simple.hbase.table=threatintel threat.intel.simple.hbase.cf=t -enrichment.simple.hbase.table=enrichment -enrichment.simple.hbase.cf=t + +##### Parallelism ##### +kafka.spout.parallelism=1 +enrichment.split.parallelism=1 +enrichment.stellar.parallelism=1 +enrichment.join.parallelism=1 +threat.intel.split.parallelism=1 +threat.intel.stellar.parallelism=1 +threat.intel.join.parallelism=1 +kafka.writer.parallelism=1 diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index 0e50f77b3a..7ee1b2e359 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -16,10 +16,11 @@ name: "enrichment" config: - topology.workers: 1 - topology.acker.executors: 0 + topology.workers: ${enrichment.workers} + topology.acker.executors: ${enrichment.acker.executors} topology.worker.childopts: ${topology.worker.childopts} topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} components: @@ -56,7 +57,7 @@ components: - id: "hostEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" constructorArgs: - - '${org.apache.metron.enrichment.host.known_hosts}' + - '${enrichment.host.known_hosts}' - id: "hostEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -231,14 +232,13 @@ components: constructorArgs: - ref: "kafkaProps" # topic name - - "enrichments" + - "${enrichment.input.topic}" - "${kafka.zk}" - ref: "fields" configMethods: - name: "setFirstPollOffsetStrategy" args: - # One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST - - "UNCOMMITTED_EARLIEST" + - "${kafka.start}" spouts: @@ -246,6 +246,7 @@ spouts: className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} bolts: # Enrichment Bolts @@ -257,6 +258,8 @@ bolts: - name: "withEnrichments" args: - ref: "enrichments" + parallelism: ${enrichment.split.parallelism} + - id: "geoEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -269,6 +272,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "stellarEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -281,6 +285,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.stellar.parallelism} - id: "hostEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" @@ -294,6 +299,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "simpleHBaseEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -306,15 +312,18 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "enrichmentJoinBolt" className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [100000] + args: [${enrichment.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.join.parallelism} + - id: "enrichmentErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -336,6 +345,8 @@ bolts: - ref: "threatIntels" - name: "withMessageFieldName" args: ["message"] + parallelism: ${threat.intel.split.parallelism} + - id: "simpleHBaseThreatIntelBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -360,15 +371,19 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.stellar.parallelism} + - id: "threatIntelJoinBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [100000] + args: [${threat.intel.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.join.parallelism} + - id: "threatIntelErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -387,6 +402,7 @@ bolts: - name: "withMessageWriter" args: - ref: "kafkaWriter" + parallelism: ${kafka.writer.parallelism} streams: diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml index a8615fbf16..e67bc54c23 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml @@ -18,9 +18,10 @@ name: "indexing" config: topology.workers: ${indexing.workers} - topology.acker.executors: ${indexing.executors} + topology.acker.executors: ${indexing.acker.executors} topology.worker.childopts: ${topology.worker.childopts} topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} components: @@ -35,7 +36,7 @@ components: - ".json" - name: "withPath" args: - - "${index.hdfs.output}" + - "${indexing.hdfs.output}" - id: "hdfsRotationPolicy" className: "${bolt.hdfs.rotation.policy}" @@ -66,7 +67,7 @@ components: configMethods: - name: "withTopic" args: - - "${index.error.topic}" + - "${indexing.error.topic}" - name: "withZkQuorum" args: - "${kafka.zk}" @@ -74,7 +75,7 @@ components: args: [ref: "kafkaWriterProps"] - id: "indexWriter" - className: "${writer.class.name}" + className: "${indexing.writer.class.name}" #kafka/zookeeper # Any kafka props for the producer go here. @@ -111,13 +112,12 @@ components: constructorArgs: - ref: "kafkaProps" # topic name - - "${index.input.topic}" + - "${indexing.input.topic}" - "${kafka.zk}" - ref: "fields" configMethods: - name: "setFirstPollOffsetStrategy" args: - # One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST - "${kafka.start}" @@ -126,6 +126,8 @@ spouts: className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} + bolts: # Indexing Bolts @@ -140,6 +142,7 @@ bolts: - name: "withMessageGetter" args: - "DEFAULT_JSON_FROM_POSITION" + parallelism: ${indexing.writer.parallelism} - id: "hdfsIndexingBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" @@ -152,6 +155,7 @@ bolts: - name: "withMessageGetter" args: - "DEFAULT_JSON_FROM_POSITION" + parallelism: ${hdfs.writer.parallelism} - id: "indexingErrorBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" From a31cfc47e0f47016b7a8aa1a1675a2e791f72dea Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 7 Jun 2017 17:15:31 -0500 Subject: [PATCH 2/9] added *.json.j2 rat exception --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 16b2499986..e0a181908e 100644 --- a/pom.xml +++ b/pom.xml @@ -283,6 +283,7 @@ **/*.md **/VERSION **/*.json + **/*.json.j2 **/*.tokens **/*.log **/*.template From 07a51b481583c792fb1db9cba291e07e292f47bb Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 8 Jun 2017 08:40:25 -0500 Subject: [PATCH 3/9] updated integration tests --- .../ElasticsearchIndexingIntegrationTest.java | 2 +- .../src/main/flux/enrichment/test.yaml | 83 +++++++++++++------ .../EnrichmentIntegrationTest.java | 34 ++++++-- .../integration/IndexingIntegrationTest.java | 15 ++-- .../src/main/config/solr.properties | 51 ++++-------- .../SolrIndexingIntegrationTest.java | 2 +- 6 files changed, 108 insertions(+), 79 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java index 87c0081bfd..54e494e857 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java @@ -102,7 +102,7 @@ public void setAdditionalProperties(Properties topologyProperties) { topologyProperties.setProperty("es.clustername", "metron"); topologyProperties.setProperty("es.port", "9300"); topologyProperties.setProperty("es.ip", "localhost"); - topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter"); + topologyProperties.setProperty("indexing.writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter"); } @Override diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml index b7fb8d4ae9..b4481ff4c8 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml @@ -16,9 +16,14 @@ name: "enrichment" config: - topology.workers: 1 + topology.workers: ${enrichment.workers} + topology.acker.executors: ${enrichment.acker.executors} + topology.worker.childopts: ${topology.worker.childopts} + topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} components: + # Enrichment - id: "stellarEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" @@ -43,7 +48,7 @@ components: - id: "hostEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" constructorArgs: - - '${org.apache.metron.enrichment.host.known_hosts}' + - '${enrichment.host.known_hosts}' - id: "hostEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -179,40 +184,44 @@ components: - id: "kafkaProps" className: "java.util.HashMap" configMethods: - - name: "put" - args: - - "value.deserializer" - - "org.apache.kafka.common.serialization.ByteArrayDeserializer" - - name: "put" - args: - - "key.deserializer" - - "org.apache.kafka.common.serialization.ByteArrayDeserializer" - - name: "put" - args: - - "group.id" - - "enrichments" + - name: "put" + args: + - "value.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "key.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "group.id" + - "enrichments" + - name: "put" + args: + - "security.protocol" + - "${kafka.security.protocol}" + # The fields to pull out of the kafka messages - id: "fields" className: "java.util.ArrayList" configMethods: - - name: "add" - args: - - "value" + - name: "add" + args: + - "value" - id: "kafkaConfig" className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder" constructorArgs: - # zookeeper hosts - - ref: "kafkaProps" - # topic name - - "enrichments" - - "${kafka.zk}" - - ref: "fields" + - ref: "kafkaProps" + # topic name + - "${enrichment.input.topic}" + - "${kafka.zk}" + - ref: "fields" configMethods: - name: "setFirstPollOffsetStrategy" args: - - "UNCOMMITTED_EARLIEST" + - "${kafka.start}" spouts: @@ -220,6 +229,7 @@ spouts: className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} bolts: # Enrichment Bolts @@ -231,6 +241,8 @@ bolts: - name: "withEnrichments" args: - ref: "enrichments" + parallelism: ${enrichment.split.parallelism} + - id: "geoEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -243,6 +255,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "stellarEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -255,6 +268,8 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.stellar.parallelism} + - id: "hostEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -267,6 +282,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "simpleHBaseEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -297,9 +313,11 @@ bolts: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [10000] + args: [${enrichment.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.join.parallelism} + - id: "enrichmentErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -309,6 +327,7 @@ bolts: args: - ref: "enrichmentErrorKafkaWriter" + # Threat Intel Bolts - id: "threatIntelSplitBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt" @@ -320,6 +339,8 @@ bolts: - ref: "threatIntels" - name: "withMessageFieldName" args: ["message"] + parallelism: ${threat.intel.split.parallelism} + - id: "simpleHBaseThreatIntelBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -344,15 +365,19 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.stellar.parallelism} + - id: "threatIntelJoinBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [10000] + args: [${threat.intel.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.join.parallelism} + - id: "threatIntelErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -371,6 +396,7 @@ bolts: - name: "withMessageWriter" args: - ref: "kafkaWriter" + parallelism: ${kafka.writer.parallelism} streams: @@ -405,6 +431,7 @@ streams: type: FIELDS args: ["key"] + - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt" from: "enrichmentSplitBolt" to: "simpleHBaseEnrichmentBolt" @@ -444,6 +471,7 @@ streams: args: ["key"] + - name: "simpleHBaseEnrichmentBolt -> join" from: "simpleHBaseEnrichmentBolt" to: "enrichmentJoinBolt" @@ -524,6 +552,7 @@ streams: type: FIELDS args: ["key"] + - name: "simpleHBaseThreatIntel -> join" from: "simpleHBaseThreatIntelBolt" to: "threatIntelJoinBolt" @@ -539,6 +568,7 @@ streams: streamId: "stellar" type: FIELDS args: ["key"] + - name: "threatIntelSplit -> threatIntelJoin" from: "threatIntelSplitBolt" to: "threatIntelJoinBolt" @@ -546,7 +576,6 @@ streams: streamId: "message" type: FIELDS args: ["key"] - #output - name: "threatIntelJoin -> output" from: "threatIntelJoinBolt" diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 963480502a..75da28932c 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -24,7 +24,6 @@ import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; @@ -32,8 +31,6 @@ import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; -import org.apache.metron.enrichment.bolt.ErrorEnrichmentBolt; -import org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt; import org.apache.metron.enrichment.converter.EnrichmentHelper; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; @@ -44,7 +41,6 @@ import org.apache.metron.hbase.TableProvider; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; -import org.apache.metron.integration.Processor; import org.apache.metron.integration.ProcessorResult; import org.apache.metron.integration.components.FluxTopologyComponent; import org.apache.metron.integration.components.KafkaComponent; @@ -128,7 +124,21 @@ public void test() throws Exception { final String threatIntelTableName = "threat_intel"; final String enrichmentsTableName = "enrichments"; final Properties topologyProperties = new Properties() {{ - setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" + + setProperty("enrichment.workers", "1"); + setProperty("enrichment.acker.executors", "0"); + setProperty("topology.worker.childopts", ""); + setProperty("topology.auto-credentials", "[]"); + setProperty("topology.max.spout.pending", ""); + setProperty("kafka.start", "UNCOMMITTED_EARLIEST"); + setProperty("kafka.security.protocol", "PLAINTEXT"); + setProperty("enrichment.input.topic", Constants.ENRICHMENT_TOPIC); + setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC); + setProperty("enrichment.error.topic", ERROR_TOPIC); + setProperty("threat.intel.error.topic", ERROR_TOPIC); + setProperty("enrichment.join.cache.size", "1000"); + setProperty("threat.intel.join.cache.size", "1000"); + + setProperty("enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]"); @@ -139,10 +149,16 @@ public void test() throws Exception { setProperty("threat.intel.simple.hbase.cf", cf); setProperty("enrichment.simple.hbase.table", enrichmentsTableName); setProperty("enrichment.simple.hbase.cf", cf); - setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC); - setProperty("enrichment.error.topic", ERROR_TOPIC); - setProperty("kafka.security.protocol", "PLAINTEXT"); - setProperty("storm.auto.credentials", "[]"); + + setProperty("kafka.spout.parallelism", "1"); + setProperty("enrichment.split.parallelism", "1"); + setProperty("enrichment.stellar.parallelism", "1"); + setProperty("enrichment.join.parallelism", "1"); + setProperty("threat.intel.split.parallelism", "1"); + setProperty("threat.intel.stellar.parallelism", "1"); + setProperty("threat.intel.join.parallelism", "1"); + setProperty("kafka.writer.parallelism", "1"); + }}; final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList() {{ diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java index da46d9309e..46b600cd95 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java @@ -117,24 +117,25 @@ public static void cleanHdfsDir(String hdfsDirStr) { @Test public void test() throws Exception { cleanHdfsDir(hdfsDir); - final String dateFormat = "yyyy.MM.dd.HH"; final List inputMessages = TestUtils.readSampleData(sampleParsedPath); final Properties topologyProperties = new Properties() {{ setProperty("kafka.start", "UNCOMMITTED_EARLIEST"); setProperty("kafka.security.protocol", "PLAINTEXT"); setProperty("storm.auto.credentials", "[]"); setProperty("indexing.workers", "1"); - setProperty("indexing.executors", "0"); - setProperty("index.input.topic", Constants.INDEXING_TOPIC); - setProperty("index.error.topic", ERROR_TOPIC); - setProperty("index.date.format", dateFormat); + setProperty("indexing.acker.executors", "0"); + setProperty("topology.max.spout.pending", ""); + setProperty("indexing.input.topic", Constants.INDEXING_TOPIC); + setProperty("indexing.error.topic", ERROR_TOPIC); setProperty("topology.auto-credentials", "[]"); //HDFS settings - setProperty("bolt.hdfs.rotation.policy", TimedRotationPolicy.class.getCanonicalName()); setProperty("bolt.hdfs.rotation.policy.count", "1"); setProperty("bolt.hdfs.rotation.policy.units", "DAYS"); - setProperty("index.hdfs.output", hdfsDir); + setProperty("indexing.hdfs.output", hdfsDir); + setProperty("kafka.spout.parallelism", "1"); + setProperty("indexing.writer.parallelism", "1"); + setProperty("hdfs.writer.parallelism", "1"); }}; setAdditionalProperties(topologyProperties); final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); diff --git a/metron-platform/metron-solr/src/main/config/solr.properties b/metron-platform/metron-solr/src/main/config/solr.properties index f0eca6c53d..7a7a091a0e 100644 --- a/metron-platform/metron-solr/src/main/config/solr.properties +++ b/metron-platform/metron-solr/src/main/config/solr.properties @@ -16,49 +16,32 @@ ##### Storm ##### indexing.workers=1 -indexing.executors=0 +indexing.acker.executors=0 topology.worker.childopts= topology.auto-credentials=[''] +topology.max.spout.pending= ##### Kafka ##### - kafka.zk=node1:2181 kafka.broker=node1:6667 -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start=UNCOMMITTED_EARLIEST kafka.security.protocol=PLAINTEXT -##### Indexing ##### -index.input.topic=indexing -index.error.topic=indexing -writer.class.name=org.apache.metron.solr.writer.SolrWriter - -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses - -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true +indexing.input.topic=indexing +indexing.error.topic=indexing +##### Indexing ##### +indexing.writer.class.name=org.apache.metron.solr.writer.SolrWriter ##### HDFS ##### - -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020 -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec -index.hdfs.output=/tmp/metron/enriched - +bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy +bolt.hdfs.rotation.policy.units=DAYS +bolt.hdfs.rotation.policy.count=1 +indexing.hdfs.output=/tmp/metron/enriched + +##### Parallelism ##### +kafka.spout.parallelism=1 +indexing.writer.parallelism=1 +hdfs.writer.parallelism=1 \ No newline at end of file diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java index c209ef346a..f47e8e89ef 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java @@ -115,7 +115,7 @@ public ProcessorResult>> getResult() { @Override public void setAdditionalProperties(Properties topologyProperties) { - topologyProperties.setProperty("writer.class.name", "org.apache.metron.solr.writer.SolrWriter"); + topologyProperties.setProperty("indexing.writer.class.name", "org.apache.metron.solr.writer.SolrWriter"); } @Override From d846ff39016d7be00780ee7f66877965594b4aab Mon Sep 17 00:00:00 2001 From: merrimanr Date: Mon, 12 Jun 2017 10:16:36 -0500 Subject: [PATCH 4/9] added tabs for REST and Management UI --- .../CURRENT/configuration/metron-env.xml | 26 +-- .../METRON/CURRENT/themes/metron_theme.json | 192 +++++++++++++++++- 2 files changed, 203 insertions(+), 15 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 0e0efda1f0..7f226298ae 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -137,7 +137,7 @@ - + metron_jdbc_platform Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql @@ -486,15 +486,21 @@ bolt_hdfs_rotation_policy - Class Name of Hdfs Rotation Policy + Class Name of HDFS Rotation Policy org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy - Hdfs Rotation Policy + HDFS Rotation Policy + + + bolt_hdfs_rotation_policy_count + HDFS Rotation Policy Count + 1 + HDFS Rotation Policy Count bolt_hdfs_rotation_policy_units - Frequency of Hdfs File Rotation + Frequency of HDFS File Rotation DAYS - Hdfs Rotation Policy Units + HDFS Rotation Policy Units value-list @@ -514,12 +520,6 @@ 1 - - bolt_hdfs_rotation_policy_count - Hdfs Rotation Policy Count - 1 - Hdfs Rotation Policy Count - indexing_error_topic Indexing Error Topic @@ -540,8 +540,8 @@ hdfs_writer_parallelism - Indexing Topology Hdfs Bolt Parallelism + Indexing Topology HDFS Bolt Parallelism 1 - Indexing Hdfs Parallelism + Indexing HDFS Parallelism diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 15388e7395..6ccd730539 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -183,7 +183,63 @@ "subsections": [ { "name": "subsection-indexing-hdfs", - "display-name": "Hdfs", + "display-name": "HDFS", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "rest", + "display-name": "REST", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-rest", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-rest", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "management-ui", + "display-name": "Management UI", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-management-ui", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-management-ui", "row-index": "0", "column-index": "0", "row-span": "1", @@ -368,6 +424,59 @@ { "config": "metron-env/bolt_hdfs_rotation_policy_count", "subsection-name": "subsection-indexing-hdfs" + }, + + { + "config": "metron-env/metron_rest_port", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_url", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_driver", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_username", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_password", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_platform", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_client_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_default_grok_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_temp_grok_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_spring_profiles_active", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jvm_flags", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_spring_options", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_management_ui_port", + "subsection-name": "subsection-management-ui" } ] }, @@ -627,12 +736,91 @@ "type": "text-field" } }, - { "config": "metron-env/metron_apps_indexed_hdfs_dir", "widget": { "type": "text-field" } + }, + + + { + "config": "metron-env/metron_rest_port", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_driver", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_username", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_password", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_platform", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_client_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_default_grok_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_temp_grok_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_spring_profiles_active", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jvm_flags", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_spring_options", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_management_ui_port", + "widget": { + "type": "text-field" + } } ] } From 2eee22b4eece5e31277550ee6d9b1d516577482e Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 21 Jun 2017 14:53:23 -0500 Subject: [PATCH 5/9] Added kafka_security_protocol --- .../METRON/CURRENT/package/templates/enrichment.properties.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 index 2aca4059d0..485b93882f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 @@ -26,7 +26,7 @@ topology.max.spout.pending={{enrichment_topology_max_spout_pending}} ##### Kafka ##### kafka.zk={{zookeeper_quorum}} kafka.broker={{kafka_brokers}} -kafka.security.protocol=PLAINTEXT +kafka.security.protocol={{kafka_security_protocol}} # One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST kafka.start={{enrichment_kafka_start}} From c14f47e53b90feccf8cf8e6832ec8cf893ad9a4d Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 21 Jun 2017 14:57:04 -0500 Subject: [PATCH 6/9] Removed redundant property and add file size units --- .../CURRENT/configuration/metron-env.xml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 7f226298ae..66f146284e 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -516,16 +516,22 @@ DAYS + + KB + + + MB + + + GB + + + TB + 1 - - indexing_error_topic - Indexing Error Topic - indexing - Indexing Error Topic - indexing_kafka_spout_parallelism Indexing Topology Kafka Spout Parallelism From 4aa754a67ace0b952e1489ac3c47bad5547021d4 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 21 Jun 2017 16:16:35 -0500 Subject: [PATCH 7/9] Added configure call to restart function --- .../METRON/CURRENT/package/scripts/enrichment_master.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index 4ab81d4c54..de3c3f8d2d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -105,6 +105,7 @@ def status(self, env): def restart(self, env): from params import params env.set_params(params) + self.configure(env) commands = EnrichmentCommands(params) commands.restart_enrichment_topology(env) From db72a2b3ad06f8be030cc0bb21249410990d925f Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 21 Jun 2017 16:17:53 -0500 Subject: [PATCH 8/9] Fixed password widget type --- .../common-services/METRON/CURRENT/themes/metron_theme.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 6ccd730539..22ca4c0d25 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -771,7 +771,7 @@ { "config": "metron-env/metron_jdbc_password", "widget": { - "type": "text-field" + "type": "password" } }, { From 1a9b19a0101ada58cb671ab224934f304df6fff8 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 22 Jun 2017 16:49:54 -0500 Subject: [PATCH 9/9] password needs to be passed to start in case of restart --- metron-interface/metron-rest/src/main/scripts/metron-rest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest index f9227a95f3..3fa1df01ed 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest @@ -145,7 +145,7 @@ case "$1" in # restart) $0 stop - $0 start + $0 start $2 ;; ##############################################################################