From d4b8ea8c148f1753932f799e3c553847cdb3b05c Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 5 Aug 2016 09:23:12 -0400 Subject: [PATCH 01/82] sample --- .../0.2.0BETA/configuration/metron.xml | 31 +++++++ .../PARSERS/0.2.0BETA/metainfo.xml | 60 +++++++++++++ .../PARSERS/0.2.0BETA/package/.hash | 1 + .../PARSERS/0.2.0BETA/package/archive.zip | Bin 0 -> 4129 bytes .../0.2.0BETA/package/scripts/params.py | 27 ++++++ .../package/scripts/parsers_master.py | 84 ++++++++++++++++++ .../HDP/2.4/services/PARSERS/metainfo.xml | 23 +++++ 7 files changed, 226 insertions(+) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml new file mode 100755 index 0000000000..018dc09600 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml @@ -0,0 +1,31 @@ + + + + + + metron_home + /usr/metron/0.2.0BETA + Metron home directory + + + parsers + bluecoat,bro,snort,squid,websphere,yaf + Metron parsers to deploy + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..ea3479d615 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -0,0 +1,60 @@ + + + + 2.0 + + + PARSERS + Parsers + Parsers + 0.2.0BETA + + + PARSER_MASTER + Parsers Master-Eligible Node + MASTER + 1 + + + PYTHON + 600 + + + + + + any + + + metron-parsers + + + + + + + PYTHON + 300 + + + parsers-env + + true + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash new file mode 100644 index 0000000000..eb3e6503db --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash @@ -0,0 +1 @@ +cae228d81fcb59299a9f086e37cf9c8dbb6aba7d \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..220aa73c79f8832080d0718c5a1a6104c7065b54 GIT binary patch literal 4129 zcmeHKPmkL~6koapA#*@Ty>NSYk&@7jQ;^_L6r{A>RikZ_Xp>Tvwz4Mn#2KwUV`e5z z)2c!}03QO0FTl0pTlEug;k_9<*|b}Vs&b-UC0UQ(|3AO?JUZGve_@BNXMgPWzxnXb zXMgSP?9k^EJTr+Fr8S+BX;zr1Tz%tEw!KF13 zz?gFgLixm-rfB_vWh>*PMTrgH8Feo;##UzgV5%%{Wy2?!B2-(2uAWX8#)wG>dt>O2L%7l#_s9Fe z?f&HY@YV!w_l}QygGv8j48vo%IviZW8{9&odb~I2> z?06alb+OdSsT#-_Q}RU2M1mg4Y{4=PnR>vrq+%$!E`*`%88j^oa#0BDD#n~jcqK6k zT;T^ZtqP2qQ45{$xL^`xQpHBOm};g4H<9lnvQ23VW7kzdum^@~R856H6WKz0O@Y6K zZbcYn+{XT_*$RS!Tdia~R|Q8Be}1>IEuwM%Zo|`U{&O6S`vLYcsx|+Tx1vVA(Hbd! z`~7_X*RT0cpMSKoL!VE#k%FXS7>wl_5#t>;Y#QJlHaK`k{P^EvLxKY@szCz*VukHe z)BthtkS7bv-yB(jBxeYRBj4&x4>+yMekS;qj&mHG2m%9|z;0Hgd z5z^&@*D6Mu$RH|uZB74ENhGPI7Scka?*%*9zv%#}v9^704<^HFLumUEycBs3Q;wsT z<4D!KP?p1LLBL)rXCY@PP1#%}EMH5hGSv);w7sfbfkbnL_=mh<>t*82Q|LH(uOyro zDuwnjsP^)reho2C<_hkGcW)j{j)#Nxr3k0`m4iv|{yiD4ZJ_AT6-rj{?oBsKU`-0R zBb?y)GH0?(nc)kacWGV0rD9|HnbcXC;R2YPbT6Ixm`hqK((ae{Lii3k+$J52Ar*`= zQR-#8I}G63`WNzATf51stH`RJusXamO)9CEgxpY=P+z>&6y1oai(@7vZk4!U;G(7g z=Wxf&X_=)x;)kP(ZPO2XP}f7%2B&BBMF`#A{OBakQFS%ZvGwQBZeyA`jBbsuLtDV( z$M8t06V7oRsmRb5Ty38`f%c>A$CwX1u46>qguSHgrkD0%QD&N@JhrQncd6aqw7q=g zth;MWiq7mUTD7Xk;+FU#N^DSt1XiJ5r`Vl&C*q_?5uyGemd;l~0o*xat4futY()_b z;bLXMmf)&>oaMCdHO?m6IjG&I~LRX9li)jSa*ZJpFHT z;$u*OdKEGsm{+4Pf^eNvGO&ube@kkHc%%zBAz(ZOV(Sp9^32@V9Z(BAh4UCGyD*zD zyuCU^vGp5Jjq}bk>IWRM+a2+i)E;}elzD=KwO&C}l+ZPfq1%Np#$8y*I1YW?kE)s6 zinyQhxYLi0-aB_;_tmfA^m /etc/yum.repos.d/local.repo") + + print 'Install packages' + self.install_packages(env) + + print 'Load parser config' + ## TODO find where to get/share the ZK configs + # Execute(params.metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }}") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + print 'Start the Master' + for parser in self.parser_list: + ## TODO + # start_cmd = params.metron_home + "/bin/start_parser_topology.sh " + parser + print 'Starting ' + parser + # Execute(start_cmd) + print 'Finished starting parser topologies' + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + parser_list = params.parsers.replace(' ', '').split('') + stop_cmd = format("storm kill ") + print 'Stop the Master' + Execute(stop_cmd) + + def status(self, env): + import params + env.set_params(params) + print 'Status of the Master' + + def restart(self, env): + import params + env.set_params(params) + print 'Restarting the Master' + +if __name__ == "__main__": + Parsers().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml new file mode 100644 index 0000000000..c0fc445d81 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml @@ -0,0 +1,23 @@ + + + 2.0 + + + PARSERS + Parsers + Parsers + 0.2.0BETA + common-services/PARSERS/0.2.0BETA + + + any + + + metron-parsers + + + + + + + From 0193f467f6b79a81479d30798525d981757d3146 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 9 Aug 2016 18:14:10 -0400 Subject: [PATCH 02/82] add parser and common-services --- metron-deployment/packaging/ambari/.gitignore | 3 + .../{metron.xml => metron-parsers.xml} | 26 +++++ .../PARSERS/0.2.0BETA/metainfo.xml | 33 ++++++- .../PARSERS/0.2.0BETA/package/.hash | 1 - .../PARSERS/0.2.0BETA/package/archive.zip | Bin 4129 -> 0 bytes .../0.2.0BETA/package/scripts/params.py | 15 ++- .../0.2.0BETA/package/scripts/params_linux.py | 89 ++++++++++++++++++ .../package/scripts/params_windows.py | 20 ++++ .../0.2.0BETA/package/scripts/parsers.py | 33 +++++++ .../package/scripts/parsers_master.py | 54 +++++++---- .../package/scripts/service_check.py | 41 ++++++++ .../HDP/2.4/services/PARSERS/metainfo.xml | 12 --- 12 files changed, 290 insertions(+), 37 deletions(-) create mode 100644 metron-deployment/packaging/ambari/.gitignore rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/{metron.xml => metron-parsers.xml} (56%) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip mode change 100755 => 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore new file mode 100644 index 0000000000..db8ded0080 --- /dev/null +++ b/metron-deployment/packaging/ambari/.gitignore @@ -0,0 +1,3 @@ +archive.zip +*.hash + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml similarity index 56% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml index 018dc09600..f067776f87 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml @@ -22,10 +22,36 @@ metron_home /usr/metron/0.2.0BETA Metron home directory + Metron home parsers bluecoat,bro,snort,squid,websphere,yaf Metron parsers to deploy + Metron parsers + + + metron_apps_hdfs_dir + /apps/metron + Metron apps HDFS dir + Metron apps HDFS dir + + + metron_zookeeper_config_dir + config/zookeeper + Metron Zookeeper config dir. Relative path to Metron home. + Metron Zookeeper config dir + + + metron_user + metron + USER + The user for Metron + Metron User + + + metron_group + metron + The group for Metron diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index ea3479d615..d33720699b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -23,12 +23,30 @@ Parsers Parsers 0.2.0BETA + PARSER_MASTER Parsers Master-Eligible Node MASTER 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + PYTHON @@ -36,25 +54,36 @@ + any + metron-common metron-parsers + PYTHON 300 + + + HDFS + KAFKA + STORM + ZOOKEEPER + + - parsers-env + metron-parsers - true + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash deleted file mode 100644 index eb3e6503db..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash +++ /dev/null @@ -1 +0,0 @@ -cae228d81fcb59299a9f086e37cf9c8dbb6aba7d \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip deleted file mode 100644 index 220aa73c79f8832080d0718c5a1a6104c7065b54..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4129 zcmeHKPmkL~6koapA#*@Ty>NSYk&@7jQ;^_L6r{A>RikZ_Xp>Tvwz4Mn#2KwUV`e5z z)2c!}03QO0FTl0pTlEug;k_9<*|b}Vs&b-UC0UQ(|3AO?JUZGve_@BNXMgPWzxnXb zXMgSP?9k^EJTr+Fr8S+BX;zr1Tz%tEw!KF13 zz?gFgLixm-rfB_vWh>*PMTrgH8Feo;##UzgV5%%{Wy2?!B2-(2uAWX8#)wG>dt>O2L%7l#_s9Fe z?f&HY@YV!w_l}QygGv8j48vo%IviZW8{9&odb~I2> z?06alb+OdSsT#-_Q}RU2M1mg4Y{4=PnR>vrq+%$!E`*`%88j^oa#0BDD#n~jcqK6k zT;T^ZtqP2qQ45{$xL^`xQpHBOm};g4H<9lnvQ23VW7kzdum^@~R856H6WKz0O@Y6K zZbcYn+{XT_*$RS!Tdia~R|Q8Be}1>IEuwM%Zo|`U{&O6S`vLYcsx|+Tx1vVA(Hbd! z`~7_X*RT0cpMSKoL!VE#k%FXS7>wl_5#t>;Y#QJlHaK`k{P^EvLxKY@szCz*VukHe z)BthtkS7bv-yB(jBxeYRBj4&x4>+yMekS;qj&mHG2m%9|z;0Hgd z5z^&@*D6Mu$RH|uZB74ENhGPI7Scka?*%*9zv%#}v9^704<^HFLumUEycBs3Q;wsT z<4D!KP?p1LLBL)rXCY@PP1#%}EMH5hGSv);w7sfbfkbnL_=mh<>t*82Q|LH(uOyro zDuwnjsP^)reho2C<_hkGcW)j{j)#Nxr3k0`m4iv|{yiD4ZJ_AT6-rj{?oBsKU`-0R zBb?y)GH0?(nc)kacWGV0rD9|HnbcXC;R2YPbT6Ixm`hqK((ae{Lii3k+$J52Ar*`= zQR-#8I}G63`WNzATf51stH`RJusXamO)9CEgxpY=P+z>&6y1oai(@7vZk4!U;G(7g z=Wxf&X_=)x;)kP(ZPO2XP}f7%2B&BBMF`#A{OBakQFS%ZvGwQBZeyA`jBbsuLtDV( z$M8t06V7oRsmRb5Ty38`f%c>A$CwX1u46>qguSHgrkD0%QD&N@JhrQncd6aqw7q=g zth;MWiq7mUTD7Xk;+FU#N^DSt1XiJ5r`Vl&C*q_?5uyGemd;l~0o*xat4futY()_b z;bLXMmf)&>oaMCdHO?m6IjG&I~LRX9li)jSa*ZJpFHT z;$u*OdKEGsm{+4Pf^eNvGO&ube@kkHc%%zBAz(ZOV(Sp9^32@V9Z(BAh4UCGyD*zD zyuCU^vGp5Jjq}bk>IWRM+a2+i)E;}elzD=KwO&C}l+ZPfq1%Np#$8y*I1YW?kE)s6 zinyQhxYLi0-aB_;_tmfA^m /etc/yum.repos.d/local.repo") - print 'Install packages' + print 'Install RPM packages' self.install_packages(env) + print 'Upload grok patterns' + parsers_init() + + print 'Setup Kafka topics' + for parser in parser_list: + Execute(format("""/usr/hdp/current/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {zookeeper_url} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""").format(parser,1,1,10 * 1024 * 1024 * 1024)) +# {{ kafka_home }}/bin/kafka-topics.sh \ +# --zookeeper {{ zookeeper_url }} \ +# --create \ +# --topic {{ item.topic }} \ +# --partitions {{ item.num_partitions }} \ +# --replication-factor {{ item.replication_factor }} \ +# --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }} + print 'Load parser config' - ## TODO find where to get/share the ZK configs - # Execute(params.metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }}") + #Execute(format("{metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {zookeeper_url}") + Execute(format("{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_url}")) def start(self, env, upgrade_type=None): import params env.set_params(params) + parser_list = params.parsers.replace(' ', '').split(',') print 'Start the Master' - for parser in self.parser_list: - ## TODO - # start_cmd = params.metron_home + "/bin/start_parser_topology.sh " + parser + for parser in parser_list: print 'Starting ' + parser - # Execute(start_cmd) + start_cmd = params.metron_home + '/bin/start_parser_topology.sh -s ' + parser + ' -z localhost:2181' + Execute(start_cmd) print 'Finished starting parser topologies' def stop(self, env, upgrade_type=None): import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split('') - stop_cmd = format("storm kill ") - print 'Stop the Master' - Execute(stop_cmd) + parser_list = params.parsers.replace(' ', '').split(',') + print 'Stopping parsers' + for parser in parser_list: + print 'Stopping ' + parser + stop_cmd = 'storm kill ' + parser + Execute(stop_cmd) + print 'Done' def status(self, env): import params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py new file mode 100755 index 0000000000..a363d65746 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from __future__ import print_function + +import sys + +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute +import subprocess + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + doc = '{"name": "Ambari Smoke test"}' + index = "ambari_smoke_test" + + print("Running parsers service check", file=sys.stdout) + exit(0) + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml old mode 100644 new mode 100755 index c0fc445d81..6280214962 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml @@ -4,20 +4,8 @@ PARSERS - Parsers - Parsers 0.2.0BETA common-services/PARSERS/0.2.0BETA - - - any - - - metron-parsers - - - - From 67096076717087126bdcf1e07d5e28d5b55a52e7 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 10 Aug 2016 18:12:04 -0400 Subject: [PATCH 03/82] finish commands. refactor --- .../0.2.0BETA/package/scripts/commands.py | 126 ++++++++++++++++++ .../package/scripts/{ => params}/params.py | 0 .../scripts/{ => params}/params_linux.py | 22 +-- .../scripts/{ => params}/params_windows.py | 0 .../0.2.0BETA/package/scripts/parsers.py | 33 ----- .../package/scripts/parsers_master.py | 87 ++++-------- .../package/scripts/service_check.py | 2 - 7 files changed, 165 insertions(+), 105 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params.py (100%) mode change 100644 => 100755 rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params_linux.py (83%) rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params_windows.py (100%) mode change 100644 => 100755 delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py new file mode 100755 index 0000000000..ed2f880b3c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format + + +# Wrap major operations and functionality in this class +class Commands: + __params = None + __parser_list = None + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__parser_list = self.__get_parsers(params) + + # get list of parsers + def __get_parsers(self, params): + return params.parsers.replace(' ', '').split(',') + + def init_parsers(self): + Logger.info( + "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, + self.__params.metron_apps_dir)) + self.__params.HdfsResource(self.__params.metron_apps_dir, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + mode=0775, + source=self.__params.local_grok_patterns_dir) + Logger.info("Done initializing parser configuration") + + def get_parser_list(self): + return self.__parser_list + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + # TODO get HDP home from params + command_template = format("""{hadoop_home_dir}/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {zookeeper_quorum} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""") + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = 10 + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating main topics for parsers") + for parser_name in self.get_parser_list(): + Logger.info("Creating topic'{}'".format(parser_name)) + Execute(command_template.format(parser_name, num_partitions, replication_factor, retention_bytes)) + Logger.info("Creating topics for error handling") + Execute(command_template.format("parser_invalid", num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format("parser_error", num_partitions, replication_factor, retention_bytes)) + Logger.info("Done creating Kafka topics") + + def init_parser_config(self): + Logger.info('Loading parser config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) + + def start_parser_topologies(self): + Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) + start_cmd_template = format("""{metron_home}/bin/start_parser_topology.sh \ + -s {} \ + -z {zookeeper_quorum}""") + for parser in self.get_parser_list(): + Logger.info('Starting ' + parser) + Execute(start_cmd_template.format(parser)) + + Logger.info('Finished starting parser topologies') + + def stop_parser_topologies(self): + Logger.info('Stopping parsers') + for parser in self.get_parser_list(): + Logger.info('Stopping ' + parser) + stop_cmd = 'storm kill ' + parser + Execute(stop_cmd) + Logger.info('Done stopping parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py old mode 100644 new mode 100755 similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py similarity index 83% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index 2a3da98150..bc982cb06b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -35,8 +35,11 @@ hostname = config['hostname'] metron_home = config['configurations']['metron-parsers']['metron_home'] parsers = config['configurations']['metron-parsers']['parsers'] +metron_user = config['configurations']['metron-parsers']['metron_user'] +metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +yum_repo_type = 'local' # hadoop params hadoop_home_dir = stack_select.get_hadoop_dir("home") @@ -44,14 +47,17 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir() # zookeeper -# zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] -zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) -zookeeper_url = format("{hostname}:{zookeeper_port}") - -# kafka -kafka_host = config['clusterHostInfo']['kafka_broker_hosts'][0] -# TODO -kafka_port = 6667 +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py old mode 100644 new mode 100755 similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py deleted file mode 100644 index 66aa6ed3dc..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - - -def parsers_init(): - import params - params.HdfsResource(params.metron_apps_dir, - type="directory", - action="create_on_execute", - owner=params.metron_user, - mode=0775, - source=params.local_grok_patterns_dir) - - -def get_parsers(params): - return params.parsers.replace(' ', '').split(',') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index dd708b6b8a..ecdd4acd03 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -17,88 +17,51 @@ """ +from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions import format from resource_management.libraries.script import Script -from parsers import parsers_init +from commands import Commands -class Parsers(Script): + +class ParsersMaster(Script): def install(self, env): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - - print 'Install the Master' - ## TODO - this will become a remote yum repo instead of local - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - Execute("echo \"[METRON-0.2.0BETA]\n" - "name=Metron 0.2.0BETA packages\n" - "baseurl=file:///localrepo\n" - "gpgcheck=0\n" - "enabled=1\" > /etc/yum.repos.d/local.repo") - - print 'Install RPM packages' + commands = Commands(params) + commands.setup_repo() + Logger.info('Install RPM packages') self.install_packages(env) - - print 'Upload grok patterns' - parsers_init() - - print 'Setup Kafka topics' - for parser in parser_list: - Execute(format("""/usr/hdp/current/kafka-broker/bin/kafka-topics.sh \ - --zookeeper {zookeeper_url} \ - --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""").format(parser,1,1,10 * 1024 * 1024 * 1024)) -# {{ kafka_home }}/bin/kafka-topics.sh \ -# --zookeeper {{ zookeeper_url }} \ -# --create \ -# --topic {{ item.topic }} \ -# --partitions {{ item.num_partitions }} \ -# --replication-factor {{ item.replication_factor }} \ -# --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }} - - print 'Load parser config' - #Execute(format("{metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {zookeeper_url}") - Execute(format("{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_url}")) + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() def start(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - print 'Start the Master' - for parser in parser_list: - print 'Starting ' + parser - start_cmd = params.metron_home + '/bin/start_parser_topology.sh -s ' + parser + ' -z localhost:2181' - Execute(start_cmd) - print 'Finished starting parser topologies' + commands = Commands(params) + commands.start_parser_topologies() def stop(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - print 'Stopping parsers' - for parser in parser_list: - print 'Stopping ' + parser - stop_cmd = 'storm kill ' + parser - Execute(stop_cmd) - print 'Done' + commands = Commands(params) + commands.stop_parser_topologies() def status(self, env): - import params + from params import params env.set_params(params) - print 'Status of the Master' + Logger.info('Status of the Master') def restart(self, env): - import params + from params import params env.set_params(params) - print 'Restarting the Master' + Logger.info('Restarting the parser topologies') + self.stop(env) + self.start(env) + Logger.info('Done restarting the parser topologies') if __name__ == "__main__": - Parsers().execute() + ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index a363d65746..9746ea4c7b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -22,8 +22,6 @@ import sys from resource_management.libraries.script import Script -from resource_management.core.resources.system import Execute -import subprocess class ServiceCheck(Script): From 09dc18b9266702e90f4a7f02ea33edac31be0317 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 12 Aug 2016 12:00:13 -0400 Subject: [PATCH 04/82] fixes --- .../PARSERS/0.2.0BETA/metainfo.xml | 2 ++ .../0.2.0BETA/package/scripts/commands.py | 6 ++++++ .../package/scripts/params/__init__.py | 0 .../package/scripts/parsers_master.py | 20 +++++++++---------- .../services/PARSERS/metainfo.xml | 0 .../services/PARSERS/role_command_order.json | 8 ++++++++ 6 files changed, 26 insertions(+), 10 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py rename metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/{2.4 => 2.3}/services/PARSERS/metainfo.xml (100%) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index d33720699b..2291f622bb 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -61,6 +61,8 @@ metron-common + + metron-parsers diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index ed2f880b3c..118ddae4c4 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -124,3 +124,9 @@ def stop_parser_topologies(self): stop_cmd = 'storm kill ' + parser Execute(stop_cmd) Logger.info('Done stopping parser topologies') + + def restart_parser_topologies(self): + Logger.info('Restarting the parser topologies') + self.stop_parser_topologies() + self.start_parser_topologies() + Logger.info('Done restarting the parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index ecdd4acd03..4d0107fb45 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -18,15 +18,12 @@ """ from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import format from resource_management.libraries.script import Script from commands import Commands class ParsersMaster(Script): - def install(self, env): from params import params env.set_params(params) @@ -34,14 +31,18 @@ def install(self, env): commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) - commands.init_parsers() - commands.init_kafka_topics() - commands.init_parser_config() + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) def start(self, env, upgrade_type=None): from params import params env.set_params(params) commands = Commands(params) + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): @@ -58,10 +59,9 @@ def status(self, env): def restart(self, env): from params import params env.set_params(params) - Logger.info('Restarting the parser topologies') - self.stop(env) - self.start(env) - Logger.info('Done restarting the parser topologies') + commands = Commands(params) + commands.restart_parser_topologies() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml rename to metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json new file mode 100755 index 0000000000..dcee302c46 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -0,0 +1,8 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "PARSER_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] + } +} From b447901c078c2b943608e2b5304fb87fb2aaabad Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 12 Aug 2016 12:18:18 -0400 Subject: [PATCH 05/82] config fixes --- .../0.2.0BETA/package/scripts/commands.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 118ddae4c4..f1e892d5d6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -80,14 +80,13 @@ def remote_repo(): def init_kafka_topics(self): Logger.info('Creating Kafka topics') - # TODO get HDP home from params - command_template = format("""{hadoop_home_dir}/kafka-broker/bin/kafka-topics.sh \ - --zookeeper {zookeeper_quorum} \ + command_template = """{}/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {} \ --create \ --topic {} \ --partitions {} \ --replication-factor {} \ - --config retention.bytes={}""") + --config retention.bytes={}""" num_partitions = 1 replication_factor = 1 retention_gigabytes = 10 @@ -95,10 +94,24 @@ def init_kafka_topics(self): Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): Logger.info("Creating topic'{}'".format(parser_name)) - Execute(command_template.format(parser_name, num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + parser_name, + num_partitions, + replication_factor, + retention_bytes)) Logger.info("Creating topics for error handling") - Execute(command_template.format("parser_invalid", num_partitions, replication_factor, retention_bytes)) - Execute(command_template.format("parser_error", num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + "parser_invalid", + num_partitions, + replication_factor, + retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + "parser_error", + num_partitions, replication_factor, + retention_bytes)) Logger.info("Done creating Kafka topics") def init_parser_config(self): @@ -108,12 +121,12 @@ def init_parser_config(self): def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) - start_cmd_template = format("""{metron_home}/bin/start_parser_topology.sh \ + start_cmd_template = """{}/bin/start_parser_topology.sh \ -s {} \ - -z {zookeeper_quorum}""") + -z {}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) - Execute(start_cmd_template.format(parser)) + Execute(start_cmd_template.format(self.__params.metron_home, parser, self.__params.zookeeper_quorum)) Logger.info('Finished starting parser topologies') From 724070de4696059b98a358d585ff724b2f2226f9 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 16 Aug 2016 10:10:27 -0400 Subject: [PATCH 06/82] incremental progress. quicklinks. kafka home --- .../PARSERS/0.2.0BETA/metainfo.xml | 18 ++++++++++-- .../0.2.0BETA/package/scripts/commands.py | 8 +++--- .../package/scripts/params/params_linux.py | 4 +++ .../0.2.0BETA/quicklinks/quicklinks.json | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 2291f622bb..77adbf8f89 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -20,14 +20,14 @@ PARSERS - Parsers + Metron Parsers Parsers 0.2.0BETA PARSER_MASTER - Parsers Master-Eligible Node + Metron Parser Topologies MASTER 1 true @@ -46,6 +46,13 @@ true + + KAFKA/KAFKA_BROKER + cluster + + true + + @@ -86,6 +93,13 @@ metron-parsers + + + quicklinks.json + true + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index f1e892d5d6..7aefef3065 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -80,7 +80,7 @@ def remote_repo(): def init_kafka_topics(self): Logger.info('Creating Kafka topics') - command_template = """{}/kafka-broker/bin/kafka-topics.sh \ + command_template = """{}/kafka-topics.sh \ --zookeeper {} \ --create \ --topic {} \ @@ -94,20 +94,20 @@ def init_kafka_topics(self): Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): Logger.info("Creating topic'{}'".format(parser_name)) - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, parser_name, num_partitions, replication_factor, retention_bytes)) Logger.info("Creating topics for error handling") - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, "parser_invalid", num_partitions, replication_factor, retention_bytes)) - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, "parser_error", num_partitions, replication_factor, diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index bc982cb06b..be91c497ed 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -19,6 +19,7 @@ """ import functools +import os from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import format @@ -42,9 +43,12 @@ yum_repo_type = 'local' # hadoop params +stack_root = Script.get_stack_root() hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin", "kafka") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json new file mode 100755 index 0000000000..ee1b225084 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} From e83964b800875ea341407e58b921f2429d0c517d Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 16 Aug 2016 16:39:43 -0400 Subject: [PATCH 07/82] Initial working version with mpack build and separate profile (build-rpms) for building rpms. --- .../packaging/ambari/metron-mpack/pom.xml | 111 +++++++++++ .../src/main/assemblies/metron-mpack.xml | 43 +++++ .../addon-services/KIBANA/4.5.1}/metainfo.xml | 0 .../KIBANA/4.5.1}/quicklinks/quicklinks.json | 0 .../KIBANA/4.5.1/configuration/kibana-env.xml | 0 .../4.5.1/configuration/kibana-site.xml | 0 .../common-services/KIBANA/4.5.1/metainfo.xml | 0 .../KIBANA/4.5.1/package/scripts/kibana.py | 0 .../4.5.1/package/scripts/kibana_master.py | 0 .../KIBANA/4.5.1/package/scripts/params.py | 0 .../src/main/resources/mpack.json | 47 +++++ metron-deployment/pom.xml | 181 ++---------------- pom.xml | 1 + 13 files changed, 220 insertions(+), 163 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/pom.xml create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml rename metron-deployment/packaging/ambari/{src/main/resources/stacks/HDP/2.4/services/KIBANA => metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1}/metainfo.xml (100%) rename metron-deployment/packaging/ambari/{src/main/resources/stacks/HDP/2.4/services/KIBANA => metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1}/quicklinks/quicklinks.json (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py (100%) rename metron-deployment/packaging/ambari/{ => metron-mpack}/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py (100%) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml new file mode 100644 index 0000000000..0a40f7588f --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml @@ -0,0 +1,111 @@ + + + 4.0.0 + org.apache.metron.packaging.mpacks + metron_mpack + 1.0.0.0-SNAPSHOT + Metron Ambari Management Pack + + + org.apache.metron + metron-deployment + 0.2.0BETA + ../../.. + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.8 + + + parse-version + validate + + parse-version + + + + regex-property + + regex-property + + + mpackVersion + ${project.version} + ^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).* + $1.$2.$3.$4 + false + + + + + + maven-resources-plugin + 2.6 + + + copy-resources + compile + + copy-resources + + + ${basedir}/target/ + + + ${basedir}/src/main/resources + + mpack.json + + true + + + + + + + + maven-assembly-plugin + + gnu + + src/main/assemblies/metron-mpack.xml + + + + + build-tarball + package + + single + + + + + + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml new file mode 100644 index 0000000000..7de489fe6e --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml @@ -0,0 +1,43 @@ + + + + + tar.gz + + true + + + src/main/resources/common-services + common-services + + + src/main/resources/addon-services + addon-services + + + + + target/mpack.json + + + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/quicklinks/quicklinks.json rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json new file mode 100644 index 0000000000..a3ba3795ce --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json @@ -0,0 +1,47 @@ +{ + "type": "full-release", + "name": "metron-ambari.mpack", + "version": "1.0.0.0", + "description": "Ambari Management Pack for Apache Metron", + "prerequisites": { + "min-ambari-version": "2.2.0.0", + "min-stack-versions": [ + { + "stack_name": "HDP", + "stack_version": "2.3.0" + } + ] + }, + "artifacts": [ + { + "name": "METRON-common-services", + "type" : "service-definitions", + "source_dir" : "common-services" + }, + { + "name" : "METRON-addon-services", + "type" : "stack-addon-service-definitions", + "source_dir": "addon-services", + "service_versions_map": [ + { + "service_name" : "KIBANA", + "service_version" : "4.5.1", + "applicable_stacks" : [ + { + "stack_name" : "HDP", + "stack_version" : "2.3" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.4" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.5" + } + ] + } + ] + } + ] +} diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml index 24b849862a..d4b05f1440 100644 --- a/metron-deployment/pom.xml +++ b/metron-deployment/pom.xml @@ -29,166 +29,21 @@ UTF-8 rpm-docker - - - - - maven-clean-plugin - 3.0.0 - - - - packaging/docker/rpm-docker/BUILD - false - - - packaging/docker/rpm-docker/BUILDROOT - false - - - packaging/docker/rpm-docker/RPMS - false - - - packaging/docker/rpm-docker/RPMS/noarch - false - - - packaging/docker/rpm-docker/SOURCES - false - - - packaging/docker/rpm-docker/SRPMS - false - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.5.0 - - - docker-build - package - - exec - - - docker - - build - -f - packaging/docker/rpm-docker/Dockerfile - -t - ${rpm.docker.tag} - . - - - - - rpm-build - package - - exec - - - docker - - run - -v - ${project.basedir}/packaging/docker/rpm-docker:/root - ${rpm.docker.tag}:latest - /bin/bash - -c - ./build.sh ${project.version} - - - - - - - maven-resources-plugin - 3.0.1 - - - copy-rpm-sources - prepare-package - - copy-resources - - - ${project.basedir}/packaging/docker/rpm-docker/SOURCES - - - ${project.basedir}/../metron-platform/metron-common/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-data-management/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-elasticsearch/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-enrichment/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-indexing/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-parsers/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-pcap-backend/target/ - - *.tar.gz - - - - ${project.basedir}/../metron-platform/metron-solr/target/ - - *.tar.gz - - - - - - - copy-rpms-target - package - - copy-resources - - - ${project.basedir}/target/RPMS - - - packaging/docker/rpm-docker/RPMS - - - - - - - - - + + + build-rpms + + packaging/docker/rpm-docker + + + + default + + true + + + packaging/ambari/metron-mpack + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index c138d2bead..220b7069d7 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ metron-analytics metron-platform + metron-deployment @ApacheMetron From edd7b60c0565ef725858aa32e299c1b20d1f481b Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 16 Aug 2016 17:01:46 -0400 Subject: [PATCH 08/82] incremental fixes --- .../configuration/metron-parsers.xml | 2 +- .../PARSERS/0.2.0BETA/metainfo.xml | 21 ++++++ .../package/scripts/custom_commands.py | 40 +++++++++++ .../package/scripts/params/params_linux.py | 2 +- .../package/scripts/parsers_master.py | 12 ++++ .../PARSERS/0.2.0BETA/service_advisor.py | 69 +++++++++++++++++++ 6 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml index f067776f87..71966eab9f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml @@ -26,7 +26,7 @@ parsers - bluecoat,bro,snort,squid,websphere,yaf + bro,snort,yaf Metron parsers to deploy Metron parsers diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 77adbf8f89..f92d424c87 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -59,6 +59,24 @@ PYTHON 600 + + + KAFKABUILD + + + PYTHON + 600 + + + + ZOOKEEPERBUILD + + + PYTHON + 600 + + + @@ -66,6 +84,9 @@ any + + java-1.8.0-openjdk-devel + metron-common diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py new file mode 100644 index 0000000000..46bb7d3d34 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +from commands import Commands + + +class CustomCommands(Script): + def kafkabuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + def zookeeperbuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_parser_config() + + +if __name__ == "__main__": + CustomCommands().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index be91c497ed..f8b8390245 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -48,7 +48,7 @@ hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin", "kafka") +kafka_bin_dir = os.path.join(kafka_home, "bin") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index 4d0107fb45..8f893c016f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -62,6 +62,18 @@ def restart(self, env): commands = Commands(params) commands.restart_parser_topologies() + def kafkabuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + def zookeeperbuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_parser_config() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py new file mode 100644 index 0000000000..2a545da333 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -0,0 +1,69 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) + + +class MetronParsersServiceAdvisor(service_advisor.ServiceAdvisor): + # colocate Metron Parser Master with KAFKA_BROKERs + def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + parsersMasterComponent = [component for component in serviceComponents if + component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"] + parsersMasterComponent = parsersMasterComponent[0] + if not stackAdvisor.isComponentHostsPopulated(parsersMasterComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: + hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"}) + if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) + + def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + masterHosts = self.getHosts(componentsList, "PARSER_MASTER") + expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) + + items = [] + + mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Metron Parsers Master must be installed on Kafka Brokers. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( + len(mismatchHosts), hostsString) + items.append( + {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'}) + + return items From 3172d9137b05f3f4c271f2a121fa4223606c8221 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 17 Aug 2016 08:14:36 -0400 Subject: [PATCH 09/82] WIP Indexing Service --- metron-deployment/packaging/ambari/.gitignore | 2 + .../configuration/metron-indexing.xml | 59 ++++++++ .../INDEXING/0.2.0BETA/metainfo.xml | 102 ++++++++++++++ .../0.2.0BETA/package/scripts/commands.py | 126 ++++++++++++++++++ .../package/scripts/indexing_master.py | 71 ++++++++++ .../0.2.0BETA/package/scripts/params.py | 29 ++++ .../0.2.0BETA/package/scripts/params_linux.py | 63 +++++++++ .../package/scripts/params_windows.py | 17 +++ .../package/scripts/service_check.py | 38 ++++++ .../0.2.0BETA/quicklinks/quicklinks.json | 28 ++++ .../INDEXING/0.2.0BETA/service_advisor.py | 67 ++++++++++ .../HDP/2.5/services/INDEXING/metainfo.xml | 11 ++ .../services/INDEXING/role_command_order.json | 8 ++ 13 files changed, 621 insertions(+) create mode 100644 metron-deployment/packaging/ambari/.gitignore create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore new file mode 100644 index 0000000000..e1ff74205b --- /dev/null +++ b/metron-deployment/packaging/ambari/.gitignore @@ -0,0 +1,2 @@ +archive.zip +*.hash \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml new file mode 100755 index 0000000000..84c2ef4e1c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml @@ -0,0 +1,59 @@ + + + + + + metron_home + /usr/metron/0.2.0BETA + Metron home directory + Metron home + + + metron_apps_hdfs_dir + /apps/metron + Metron apps HDFS dir + Metron apps HDFS dir + + + metron_zookeeper_config_dir + config/zookeeper + Metron Zookeeper config dir. Relative path to Metron home. + Metron Zookeeper config dir + + + metron_user + metron + USER + The user for Metron + Metron User + + + metron_group + metron + The group for Metron + + + metron_indexing_topology + indexing + The Storm topology name for Indexing + + + metron_indexing_topic_retention + 10 + Topic retention, in gb + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..3a57cc1781 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml @@ -0,0 +1,102 @@ + + + + 2.0 + + + INDEXING + Indexing + Indexing + 0.2.0BETA + + + + INDEXING_MASTER + Indexing Master-Eligible Node + MASTER + 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + KAFKA/KAFKA_BROKER + cluster + + true + + + + + + PYTHON + 600 + + + + + + + any + + + metron-common + + + metron-indexing + + + + + + + + PYTHON + 300 + + + + HDFS + KAFKA + STORM + ZOOKEEPER + + + + metron-indexing + + + + + quicklinks.json + true + + + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py new file mode 100755 index 0000000000..759ad9b204 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format + + +# Wrap major operations and functionality in this class +class Commands: + __params = None + __indexing = None + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__indexing = params.metron_indexing_topology + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + command_template = """{}/kafka-topics.sh \ + --zookeeper {} \ + --create \ + --if-not-exists \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""" + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = self.__params.metron_indexing_topic_retention + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating topics for indexing") + + Logger.info("STACK HOME DIR: " + self.__params.stack_root) + Logger.info("KAFKA HOME DIR: " + self.__params.kafka_home) + Logger.info("KAFKA BIN DIR: " + self.__params.kafka_bin_dir) + + Logger.info("Creating topic'{}'".format(self.__indexing)) + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + self.__indexing, + num_partitions, + replication_factor, + retention_bytes)) + #TODO Error Handling topologies + # Logger.info("Creating topics for error handling") + # Execute(command_template.format(self.__params.hadoop_home_dir, + # self.__params.zookeeper_quorum, + # "parser_invalid", + # num_partitions, + # replication_factor, + # retention_bytes)) + # Execute(command_template.format(self.__params.hadoop_home_dir, + # self.__params.zookeeper_quorum, + # "parser_error", + # num_partitions, replication_factor, + # retention_bytes)) + Logger.info("Done creating Kafka topics") + + def init_indexing_config(self): + Logger.info('Loading indexing config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) + + def start_indexing_topology(self): + Logger.info("Starting Metron indexing topology: {}".format(self.__indexing)) + start_cmd_template = """{}/bin/start_indexing_topology.sh \ + -s {} \ + -z {}""" + Logger.info('Starting ' + self.__indexing) + Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum)) + + Logger.info('Finished starting indexing topology') + + def stop_indexing_topology(self): + Logger.info('Stopping ' + self.__indexing) + stop_cmd = 'storm kill ' + self.__indexing + Execute(stop_cmd) + Logger.info('Done stopping indexing topologies') + + def restart_indexing_topology(self): + Logger.info('Restarting the indexing topologies') + self.stop_indexing_topology() + self.start_indexing_topology() + Logger.info('Done restarting the indexing topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py new file mode 100755 index 0000000000..30eb671c56 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py @@ -0,0 +1,71 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.logger import Logger +from resource_management.libraries.script import Script + +from commands import Commands + + +class Indexing(Script): + def install(self, env): + import params + env.set_params(params) + commands = Commands(params) + commands.setup_repo() + Logger.info('Install RPM packages') + self.install_packages(env) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + commands.init_indexing_config() + commands.start_indexing_topology() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.stop_indexing_topology() + + def status(self, env): + import params + env.set_params(params) + Logger.info('Status of the Master') + + def restart(self, env): + import params + env.set_params(params) + commands = Commands(params) + commands.restart_indexing_topology() + + def kafkabuild(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + def zookeeperbuild(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.init_indexing_config() + + +if __name__ == "__main__": + Indexing().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py new file mode 100755 index 0000000000..cba46b602b --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.expect import expect + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +java_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py new file mode 100755 index 0000000000..faf78aaca6 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +hostname = config['hostname'] +metron_home = config['configurations']['metron-indexing']['metron_home'] +metron_zookeeper_config_dir = config['configurations']['metron-indexing']['metron_zookeeper_config_dir'] +metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology'] +yum_repo_type = 'local' + +# hadoop params +hadoop_home_dir = stack_select.get_hadoop_dir("home") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + +# Kafka +stack_root = Script.get_stack_root() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") +metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention'] + +# zookeeper +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort + +# kafka +kafka_host = config['clusterHostInfo']['kafka_broker_hosts'][0] +# TODO +kafka_port = 6667 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py new file mode 100755 index 0000000000..b5828d227c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py @@ -0,0 +1,17 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +raise NotImplementedError diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py new file mode 100755 index 0000000000..fec2a30512 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import print_function + +import sys + +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute +import subprocess + + +class ServiceCheck(Script): + def service_check(self, env): + # import params + # env.set_params(params) + # + # doc = '{"name": "Ambari Smoke test"}' + # index = "ambari_smoke_test" + # + # print("Running parsers service check", file=sys.stdout) + exit(0) + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json new file mode 100755 index 0000000000..492f0a0bd8 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py new file mode 100755 index 0000000000..351250887a --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py @@ -0,0 +1,67 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) + + +class MetronIndexingServiceAdvisor(service_advisor.ServiceAdvisor): + # colocate Metron Parser Master with KAFKA_BROKERs + def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + indexingMasterComponent = [component for component in serviceComponents if + component["StackServiceComponents"]["component_name"] == "INDEXING_MASTER"] + indexingMasterComponent = indexingMasterComponent[0] + if not stackAdvisor.isComponentHostsPopulated(indexingMasterComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "INDEXING_MASTER"} not in hostComponents: + hostsComponentsMap[hostName].append({"name": "INDEXING_MASTER"}) + if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "INDEXING_MASTER"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "INDEXING_MASTER"}) + + def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + masterHosts = self.getHosts(componentsList, "INDEXING_MASTER") + expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) + + items = [] + + mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Metron Parsers Master must be installed on Kafka Brokers. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( + len(mismatchHosts), hostsString) + items.append( + {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'INDEXING_MASTER'}) + + return items diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml new file mode 100755 index 0000000000..c7b407973e --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml @@ -0,0 +1,11 @@ + + + 2.0 + + + INDEXING + 0.2.0BETA + common-services/INDEXING/0.2.0BETA + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json new file mode 100755 index 0000000000..aff8bf1eb0 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json @@ -0,0 +1,8 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] + } +} \ No newline at end of file From 955680b20c6684c30631257a1068791a52e92a0a Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 17 Aug 2016 17:09:44 -0400 Subject: [PATCH 10/82] check service configured on start. fix zk load. fix topology start --- .../PARSERS/0.2.0BETA/metainfo.xml | 3 -- .../0.2.0BETA/package/scripts/commands.py | 34 +++++++++++++--- .../package/scripts/custom_commands.py | 40 ------------------- .../package/scripts/params/params_linux.py | 15 +++++++ .../package/scripts/parsers_master.py | 14 +++++-- .../package/templates/metron-global.json | 1 + .../PARSERS/0.2.0BETA/service_advisor.py | 0 7 files changed, 56 insertions(+), 51 deletions(-) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index f92d424c87..70d3378c41 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -84,9 +84,6 @@ any - - java-1.8.0-openjdk-devel - metron-common diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 7aefef3065..91e3d1d9c9 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -18,8 +18,11 @@ """ +import os + from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import Template from resource_management.libraries.functions import format @@ -27,17 +30,28 @@ class Commands: __params = None __parser_list = None + __configured = False def __init__(self, params): if params is None: raise ValueError("params argument is required for initialization") self.__params = params self.__parser_list = self.__get_parsers(params) + self.__configured = os.path.isfile(self.__params.configured_flag_file) # get list of parsers def __get_parsers(self, params): return params.parsers.replace(' ', '').split(',') + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, @@ -48,6 +62,12 @@ def init_parsers(self): owner=self.__params.metron_user, mode=0775, source=self.__params.local_grok_patterns_dir) + + Logger.info("Creating global.json file") + File(self.__params.metron_zookeeper_config_path + '/global.json', + content=Template("metron-global.json"), + owner=self.__params.metron_user, + mode=0775) Logger.info("Done initializing parser configuration") def get_parser_list(self): @@ -117,16 +137,20 @@ def init_kafka_topics(self): def init_parser_config(self): Logger.info('Loading parser config into ZooKeeper') Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) start_cmd_template = """{}/bin/start_parser_topology.sh \ - -s {} \ - -z {}""" + -k {} \ + -z {} \ + -s {}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) - Execute(start_cmd_template.format(self.__params.metron_home, parser, self.__params.zookeeper_quorum)) + Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, + self.__params.zookeeper_quorum, parser)) Logger.info('Finished starting parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py deleted file mode 100644 index 46bb7d3d34..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -from commands import Commands - - -class CustomCommands(Script): - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - - def zookeeperbuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_parser_config() - - -if __name__ == "__main__": - CustomCommands().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index f8b8390245..539dbf226c 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -40,6 +40,8 @@ metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +#metron_global_config_content = config['configurations']['global.json']['content'] yum_repo_type = 'local' # hadoop params @@ -63,11 +65,24 @@ # last port config zookeeper_quorum += ':' + zookeeper_clientPort +# Kafka +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] local_grok_patterns_dir = format("{metron_home}/patterns") hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") + # for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] hostname = config["hostname"] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index 8f893c016f..c28ee99745 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -40,9 +40,11 @@ def start(self, env, upgrade_type=None): from params import params env.set_params(params) commands = Commands(params) - commands.init_parsers() - commands.init_kafka_topics() - commands.init_parser_config() + if not commands.is_configured(): + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() + commands.set_configured() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): @@ -74,6 +76,12 @@ def zookeeperbuild(self, env, upgrade_type=None): commands = Commands(params) commands.init_parser_config() + def starttopologies(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.start_parser_topologies() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json @@ -0,0 +1 @@ +{} diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py old mode 100644 new mode 100755 From 310550291051f7d649f8f4a51a80ab722758837c Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 18 Aug 2016 16:21:33 -0400 Subject: [PATCH 11/82] service check --- .../0.2.0BETA/package/scripts/commands.py | 53 +++++++++++++++++++ .../package/scripts/params/params_linux.py | 1 - .../package/scripts/service_check.py | 11 ++-- .../PARSERS/0.2.0BETA/service_advisor.py | 11 ++-- .../services/PARSERS/role_command_order.json | 20 ++++--- 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 91e3d1d9c9..60bd3766ee 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -19,6 +19,9 @@ """ import os +import re +import subprocess +import time from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File @@ -52,6 +55,14 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) + # Possible storm topology status states + # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html + class StormStatus: + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + KILLED = "KILLED" + REBALANCING = "REBALANCING" + def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, @@ -165,5 +176,47 @@ def stop_parser_topologies(self): def restart_parser_topologies(self): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() + attempt_count = 0 + while self.topologies_active(): + if attempt_count > 2: + raise Exception("Unable to stop topologies") + attempt_count += 1 + time.sleep(10) self.start_parser_topologies() Logger.info('Done restarting the parser topologies') + + def topologies_active(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + all_active = False + if stdout_lines: + all_active = True + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + parser_found=False + is_active=False + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + status = items[1] + parser_found=True + is_active = self.__is_active(status) + all_active &= parser_found and is_active + return all_active + + def __get_status_lines(self, lines): + status_lines=[] + do_stat = False + skipped = 0 + for line in lines: + if line.startswith("Topology_name"): + do_stat = True + if do_stat and skipped == 2: + status_lines += [line] + elif do_stat: + skipped += 1 + return status_lines + + def __is_active(self, status): + return status == self.StormStatus.ACTIVE diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index 539dbf226c..b995e722b5 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -41,7 +41,6 @@ metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -#metron_global_config_content = config['configurations']['global.json']['content'] yum_repo_type = 'local' # hadoop params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 9746ea4c7b..6fcce1680a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -19,21 +19,20 @@ """ from __future__ import print_function -import sys - from resource_management.libraries.script import Script +from commands import Commands + class ServiceCheck(Script): def service_check(self, env): import params env.set_params(params) + commands = Commands(params) + all_found = commands.topologies_active() - doc = '{"name": "Ambari Smoke test"}' - index = "ambari_smoke_test" + exit(int(all_found)) - print("Running parsers service check", file=sys.stdout) - exit(0) if __name__ == "__main__": ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py index 2a545da333..ddf58d5d9a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -32,13 +32,12 @@ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) -class MetronParsersServiceAdvisor(service_advisor.ServiceAdvisor): +class PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): # colocate Metron Parser Master with KAFKA_BROKERs - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def TODO_colocateService(self, hostsComponentsMap, serviceComponents): parsersMasterComponent = [component for component in serviceComponents if - component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"] - parsersMasterComponent = parsersMasterComponent[0] - if not stackAdvisor.isComponentHostsPopulated(parsersMasterComponent): + component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0] + if not self.isComponentHostsPopulated(parsersMasterComponent): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: @@ -46,7 +45,7 @@ def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def TODO_getServiceComponentLayoutValidations(self, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json index dcee302c46..1d35b16033 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -1,8 +1,16 @@ { - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "PARSER_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] - } + "_comment": "Record format:", + "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps": { + "_comment": "dependencies for Metron Parsers", + "PARSER_MASTER-START": [ + "NAMENODE-START", + "ZOOKEEPER_SERVER-START", + "KAFKA_BROKER-START", + "STORM_REST_API-START" + ], + "PARSER_MASTER_SERVICE_CHECK-SERVICE_CHECK": [ + "PARSER_MASTER-START" + ] + } } From 2a308376fb57ceaf2c875c0f0fb754c372d74f6f Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 18 Aug 2016 17:40:12 -0400 Subject: [PATCH 12/82] trying to fix service check --- .../PARSERS/0.2.0BETA/package/scripts/params/params_linux.py | 3 +-- .../PARSERS/0.2.0BETA/package/scripts/service_check.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index b995e722b5..d9fdc6f857 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -39,7 +39,7 @@ metron_user = config['configurations']['metron-parsers']['metron_user'] metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' yum_repo_type = 'local' @@ -81,7 +81,6 @@ local_grok_patterns_dir = format("{metron_home}/patterns") hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") - # for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] hostname = config["hostname"] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 6fcce1680a..484b62806b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,7 +26,7 @@ class ServiceCheck(Script): def service_check(self, env): - import params + from params import params env.set_params(params) commands = Commands(params) all_found = commands.topologies_active() From e9abe68bba0b1b95302c99a29f5fb872dd60f1c0 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 11:25:07 -0400 Subject: [PATCH 13/82] cleanup/refactor. fix service check ordering --- .../PARSERS/0.2.0BETA/metainfo.xml | 23 +--- .../0.2.0BETA/package/scripts/commands.py | 41 ++++--- .../package/scripts/params/params_linux.py | 4 +- .../package/scripts/params/status_params.py | 113 ++++++++++++++++++ .../{parsers_master.py => parser_master.py} | 30 ++++- .../package/scripts/service_check.py | 16 ++- .../package/templates/metron-global.json | 0 .../PARSERS/0.2.0BETA/service_advisor.py | 2 +- .../services/PARSERS/role_command_order.json | 3 +- 9 files changed, 185 insertions(+), 47 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{parsers_master.py => parser_master.py} (74%) mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 70d3378c41..f1e5ab7d90 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -55,28 +55,13 @@ - + PYTHON 600 - - - KAFKABUILD - - - PYTHON - 600 - - - - ZOOKEEPERBUILD - - - PYTHON - 600 - - - + + metron-parsers + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 60bd3766ee..6b33e55a55 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -177,36 +177,49 @@ def restart_parser_topologies(self): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() attempt_count = 0 - while self.topologies_active(): + while self.topologies_exist(): if attempt_count > 2: - raise Exception("Unable to stop topologies") + raise Exception("Unable to kill topologies") attempt_count += 1 time.sleep(10) self.start_parser_topologies() Logger.info('Done restarting the parser topologies') - def topologies_active(self): + def topologies_exist(self): cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = cmd_open.communicate() stdout_lines = stdout.splitlines() - all_active = False if stdout_lines: - all_active = True status_lines = self.__get_status_lines(stdout_lines) for parser in self.get_parser_list(): - parser_found=False - is_active=False + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + return True + return False + + def topologies_running(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + all_running = False + if stdout_lines: + all_running = True + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + parser_found = False + is_running = False for line in status_lines: items = re.sub('[\s]+', ' ', line).split() if items and items[0] == parser: status = items[1] - parser_found=True - is_active = self.__is_active(status) - all_active &= parser_found and is_active - return all_active + parser_found = True + is_running = self.__is_running(status) + all_running &= parser_found and is_running + return all_running def __get_status_lines(self, lines): - status_lines=[] + status_lines = [] do_stat = False skipped = 0 for line in lines: @@ -218,5 +231,5 @@ def __get_status_lines(self, lines): skipped += 1 return status_lines - def __is_active(self, status): - return status == self.StormStatus.ACTIVE + def __is_running(self, status): + return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index d9fdc6f857..de7dfc9ff2 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -34,12 +34,14 @@ config = Script.get_config() hostname = config['hostname'] +#print(config['configurations']) metron_home = config['configurations']['metron-parsers']['metron_home'] parsers = config['configurations']['metron-parsers']['parsers'] metron_user = config['configurations']['metron-parsers']['metron_user'] metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' yum_repo_type = 'local' diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py new file mode 100755 index 0000000000..9653d07feb --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import functools +import os + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +hostname = config['hostname'] + +metron_home = config['configurations']['metron-parsers']['metron_home'] +parsers = config['configurations']['metron-parsers']['parsers'] +metron_user = config['configurations']['metron-parsers']['metron_user'] +metron_group = config['configurations']['metron-parsers']['metron_group'] +metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') +configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +yum_repo_type = 'local' + +# hadoop params +stack_root = Script.get_stack_root() +hadoop_home_dir = stack_select.get_hadoop_dir("home") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") + +# zookeeper +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort + +# Kafka +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + +metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] + +local_grok_patterns_dir = format("{metron_home}/patterns") +hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") + +# for create_hdfs_directory +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") + +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + hadoop_bin_dir=hadoop_bin_dir, + hadoop_conf_dir=hadoop_conf_dir, + principal_name=hdfs_principal_name, + hdfs_site=hdfs_site, + default_fs=default_fs, + immutable_paths=get_not_managed_resources(), + dfs_type=dfs_type +) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py similarity index 74% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py index c28ee99745..1290627567 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py @@ -17,13 +17,19 @@ """ +from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script from commands import Commands -class ParsersMaster(Script): +class ParserMaster(Script): + def get_component_name(self): + # TODO add this at some point - currently will cause problems with hdp-select + #return "parser-master" + pass + def install(self, env): from params import params env.set_params(params) @@ -54,9 +60,23 @@ def stop(self, env, upgrade_type=None): commands.stop_parser_topologies() def status(self, env): - from params import params - env.set_params(params) - Logger.info('Status of the Master') + from params import status_params + env.set_params(status_params) + commands = Commands(status_params) + if not commands.topologies_running(): + raise ComponentIsNotRunning() + +# from params import status_params +# env.set_params(params) +# commands = Commands(params) +# if not commands.topologies_running(): +# raise ComponentIsNotRunning() + +# from params import status_params +# env.set_params(status_params) +# commands = Commands(params) +# if not commands.topologies_running(): +# raise ComponentIsNotRunning() def restart(self, env): from params import params @@ -84,4 +104,4 @@ def starttopologies(self, env, upgrade_type=None): if __name__ == "__main__": - ParsersMaster().execute() + ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 484b62806b..f77a09f8f6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,12 +26,16 @@ class ServiceCheck(Script): def service_check(self, env): - from params import params - env.set_params(params) - commands = Commands(params) - all_found = commands.topologies_active() - - exit(int(all_found)) + print("service_check CALLED") +# from params import params +# env.set_params(params) +# from params import params +# env.set_params(params) +# commands = Commands(params) +# all_found = commands.topologies_active() +# +# exit(int(all_found)) + exit(0) if __name__ == "__main__": diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json old mode 100644 new mode 100755 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py index ddf58d5d9a..84b466a353 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -32,7 +32,7 @@ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) -class PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): +class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): # colocate Metron Parser Master with KAFKA_BROKERs def TODO_colocateService(self, hostsComponentsMap, serviceComponents): parsersMasterComponent = [component for component in serviceComponents if diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json index 1d35b16033..c8616469b6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -9,7 +9,8 @@ "KAFKA_BROKER-START", "STORM_REST_API-START" ], - "PARSER_MASTER_SERVICE_CHECK-SERVICE_CHECK": [ + "_comment": "_SERVICE_CHECK-SERVICE_CHECK", + "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [ "PARSER_MASTER-START" ] } From 8c1a093807c2ee18a6c9f315510bce2c2d0baf69 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 12:07:02 -0400 Subject: [PATCH 14/82] fix service check --- .../0.2.0BETA/package/scripts/service_check.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index f77a09f8f6..6ef7e4aa14 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,16 +26,14 @@ class ServiceCheck(Script): def service_check(self, env): - print("service_check CALLED") -# from params import params -# env.set_params(params) -# from params import params -# env.set_params(params) -# commands = Commands(params) -# all_found = commands.topologies_active() -# -# exit(int(all_found)) - exit(0) + from params import params + env.set_params(params) + commands = Commands(params) + all_found = commands.topologies_running() + if all_found: + exit(0) + else: + exit(1) if __name__ == "__main__": From ffae457b34e94f5fdbf22a638c62bf69d032057a Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 17:34:14 -0400 Subject: [PATCH 15/82] METRON-383 Ambari install for Metron Parser topologies. --- .../package/scripts/params/__init__.py | 18 ++++ .../package/scripts/params/status_params.py | 95 +----------------- .../package/scripts/parser_master.py | 32 +----- .../PARSERS/0.2.0BETA/service_advisor.pyc | Bin 0 -> 3700 bytes .../HDP/2.3/services/PARSERS/metainfo.xml | 16 +++ 5 files changed, 37 insertions(+), 124 deletions(-) create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py index e69de29bb2..242460e266 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py index 9653d07feb..ee83414f0c 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py @@ -18,96 +18,5 @@ """ -import functools -import os - -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import format -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.script import Script - -# server configurations -config = Script.get_config() - -hostname = config['hostname'] - -metron_home = config['configurations']['metron-parsers']['metron_home'] -parsers = config['configurations']['metron-parsers']['parsers'] -metron_user = config['configurations']['metron-parsers']['metron_user'] -metron_group = config['configurations']['metron-parsers']['metron_group'] -metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -yum_repo_type = 'local' - -# hadoop params -stack_root = Script.get_stack_root() -hadoop_home_dir = stack_select.get_hadoop_dir("home") -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hadoop_conf_dir = conf_select.get_hadoop_conf_dir() -kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin") - -# zookeeper -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -has_zk_host = not len(zk_hosts) == 0 -zookeeper_quorum = None -if has_zk_host: - if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: - zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] - else: - zookeeper_clientPort = '2181' - zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) - # last port config - zookeeper_quorum += ':' + zookeeper_clientPort - -# Kafka -kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) -has_kafka_host = not len(kafka_hosts) == 0 -kafka_brokers = None -if has_kafka_host: - if 'port' in config['configurations']['kafka-broker']: - kafka_broker_port = config['configurations']['kafka-broker']['port'] - else: - kafka_broker_port = '6667' - kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) - kafka_brokers += ':' + kafka_broker_port - -metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] - -local_grok_patterns_dir = format("{metron_home}/patterns") -hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") - -# for create_hdfs_directory -security_enabled = config['configurations']['cluster-env']['security_enabled'] -hostname = config["hostname"] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) -hdfs_site = config['configurations']['hdfs-site'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] -dfs_type = default("/commandParams/dfs_type", "") - -# create partial functions with common arguments for every HdfsResource call -# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled=security_enabled, - keytab=hdfs_user_keytab, - kinit_path_local=kinit_path_local, - hadoop_bin_dir=hadoop_bin_dir, - hadoop_conf_dir=hadoop_conf_dir, - principal_name=hdfs_principal_name, - hdfs_site=hdfs_site, - default_fs=default_fs, - immutable_paths=get_not_managed_resources(), - dfs_type=dfs_type -) +parsers = "bro,yaf,snort" +configured_flag_file = "" diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py index 1290627567..4a5b910918 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py @@ -27,7 +27,7 @@ class ParserMaster(Script): def get_component_name(self): # TODO add this at some point - currently will cause problems with hdp-select - #return "parser-master" + # return "parser-master" pass def install(self, env): @@ -66,42 +66,12 @@ def status(self, env): if not commands.topologies_running(): raise ComponentIsNotRunning() -# from params import status_params -# env.set_params(params) -# commands = Commands(params) -# if not commands.topologies_running(): -# raise ComponentIsNotRunning() - -# from params import status_params -# env.set_params(status_params) -# commands = Commands(params) -# if not commands.topologies_running(): -# raise ComponentIsNotRunning() - def restart(self, env): from params import params env.set_params(params) commands = Commands(params) commands.restart_parser_topologies() - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - - def zookeeperbuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_parser_config() - - def starttopologies(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.start_parser_topologies() - if __name__ == "__main__": ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f4735c2c1f3b7fb3eec8f58a9b6d9a631ccb277 GIT binary patch literal 3700 zcmcgu&2k&Z5$**@kXn)=_2)!!;u!5>^N*N--2t^T8b|KMXm5eN~18~8` zF0?Zcg%WW{DLJPq50FdVA+M8L4!NdsOeJ6UE+N`EJ0N#ZNOX+Rp|C`;L&uop^Aa7GF6L!AE|V_NXo*J47n2I<6?*6J|MBVt z@e1ix3d{5k+uchuF>o(((9zxC|qXBA3vw#4bl#cu2Kk@m}6;$SDCm;14vt< z<1IRZiof~#DEXY0^ffwlDf=5a$JfcxWg6X}cSKION~4=B080J(apKU|AvD^e(JdOl zESG*y%=hB>1Gd+%vMbELN<{9HsIL>FLuFHCPqb=Gf<9h;I7qRrWS-a7pH<@ruA4S*3^{pwHDd!!%l0bsRmK3-7qrw zCLDbh*%L5CMxCYEsTx2|5QY&S8pJ9}Kp2Y(Zl;Gp7DmYshfZc$G(52?Jxg?EPNE4; z>aYd=-h~<_mnsLszF9gg)W2kD;edPUIi#?e4{F=03I_L1v2^F|FJ$XDn5iVSYHHXV zXv=bye$&?z3oW72coIiJ(${WbkyeEWsP+PfvxKx~126|-fl3GBnSxCP)^!zrC)Q5B ztk=)Z&T4@Kq?TsGdYn7g)StkOZNGURs=Ds8B-X~LOuw2&8NAt>sbB&{`auus#le|k ze{wt|$T<5fgKv}JJ!Oh0e=8b!fZ0PATEW!I;ewBXMBQols^#BNj~afjRJ<$CQj|VV)3L*o?m5U`>fVV@$BGj{()ggbY7y}mx|_PIxlmbyN!Zy8hB;*ZPH+A~=O|7negr9hqWLm^pD%;ALJ`zi=34gf zyh5iH%Kj`QoKlchm{jnON|zQP+)3y=0>FK6iKmcxun=FpPp~@3`8$)Ckk{_qWny2Yx8V6Q~!h5P`##X?F#MDVB(@c+nUr|$djE)DE(Raf8 z7t+i@FtOJ_EJIubRVW_@kyICGpPLPwUEdV$^gb`AB2;ae|sy_`TdAz6^#Pvf)? z&-gjUKLK5mj+_nWXU?W`!*QKYoe%K{gxU0EJObfuWB@}LKm%}hUv6E%7!YRIt`Q*1 z3`;=gPXwLIn1QWAzzK&?r`9;9)co^c(2yFFKYUe(-H zVJgkmpp2WrOjEJl0aBxiJ+85F{urr!;t!9b((_w#c{ zX_N?aevk3k`&GuZ$A!h?y`mOdZbYpjP*M&SYMCdvGVznMOnWiUxdL(6zFp3QSJqhQ zB91wk&zG}yaJRFT?}B5g7`b^j@Quc-`!Fnzq|iMsrBWf$YH=OwpXBVkU8MQ2JsEmG z#)Gc)9OVEp+h8^yVIb#Y=jYBR&U)#(^TSf5 z^r3SdMW=$Vit}kHFX$ZYu3)~~m0am|$7wi?b+_xWcf2n!ytnatHq_%p`zap7v96S^ ztW|%2iJ+j{4bwi9WJux$CD#-LoI8Wfg1+vd^9c+lfjyBOVU$U$WAq0-BlO%}zRec) z`a53hX{Wo}^1M4NiV@>?8aw+w(*!Xn`ptHyyVrWslmk*UHaUJUk#)ZR6kZc`p1$b% zhtIs7rpLD!AvNXZjBK_{(n6a6L$-@MwZ|3 + 2.0 From d147e24097a76032b64f705456e6f3cfc335bff3 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 17 Aug 2016 08:50:06 -0400 Subject: [PATCH 16/82] Metron Indexing Service in Ambari --- metron-deployment/packaging/ambari/.gitignore | 2 +- .../configuration/metron-indexing.xml | 13 +--- .../INDEXING/0.2.0BETA/metainfo.xml | 5 +- .../0.2.0BETA/package/scripts/commands.py | 65 +++++++++++-------- .../package/scripts/indexing_master.py | 13 ++-- .../0.2.0BETA/package/scripts/params.py | 4 +- .../0.2.0BETA/package/scripts/params_linux.py | 35 +++++----- .../package/scripts/service_check.py | 23 ++++--- .../0.2.0BETA/quicklinks/quicklinks.json | 2 +- .../INDEXING/0.2.0BETA/service_advisor.py | 18 +++-- .../HDP/2.4/services/INDEXING/metainfo.xml | 26 ++++++++ .../services/INDEXING/role_command_order.json | 5 +- .../HDP/2.5/services/INDEXING/metainfo.xml | 11 ---- .../src/main/config/elasticsearch.properties | 5 +- 14 files changed, 122 insertions(+), 105 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml rename metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/{2.5 => 2.4}/services/INDEXING/role_command_order.json (67%) delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore index e1ff74205b..e708548071 100644 --- a/metron-deployment/packaging/ambari/.gitignore +++ b/metron-deployment/packaging/ambari/.gitignore @@ -1,2 +1,2 @@ archive.zip -*.hash \ No newline at end of file +*.hash diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml index 84c2ef4e1c..3a1c8f79df 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml @@ -28,12 +28,6 @@ Metron apps HDFS dir Metron apps HDFS dir - - metron_zookeeper_config_dir - config/zookeeper - Metron Zookeeper config dir. Relative path to Metron home. - Metron Zookeeper config dir - metron_user metron @@ -51,9 +45,4 @@ indexing The Storm topology name for Indexing - - metron_indexing_topic_retention - 10 - Topic retention, in gb - - \ No newline at end of file + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml index 3a57cc1781..d2dce5ac3f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml @@ -70,6 +70,9 @@ metron-indexing + + metron-elasticsearch + @@ -99,4 +102,4 @@ - \ No newline at end of file + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py index 759ad9b204..b47517f185 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py @@ -15,9 +15,11 @@ limitations under the License. """ +import subprocess +import time + from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import format # Wrap major operations and functionality in this class @@ -61,21 +63,16 @@ def init_kafka_topics(self): command_template = """{}/kafka-topics.sh \ --zookeeper {} \ --create \ - --if-not-exists \ --topic {} \ --partitions {} \ --replication-factor {} \ --config retention.bytes={}""" num_partitions = 1 replication_factor = 1 - retention_gigabytes = self.__params.metron_indexing_topic_retention + retention_gigabytes = 10 retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for indexing") - Logger.info("STACK HOME DIR: " + self.__params.stack_root) - Logger.info("KAFKA HOME DIR: " + self.__params.kafka_home) - Logger.info("KAFKA BIN DIR: " + self.__params.kafka_bin_dir) - Logger.info("Creating topic'{}'".format(self.__indexing)) Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, @@ -83,29 +80,11 @@ def init_kafka_topics(self): num_partitions, replication_factor, retention_bytes)) - #TODO Error Handling topologies - # Logger.info("Creating topics for error handling") - # Execute(command_template.format(self.__params.hadoop_home_dir, - # self.__params.zookeeper_quorum, - # "parser_invalid", - # num_partitions, - # replication_factor, - # retention_bytes)) - # Execute(command_template.format(self.__params.hadoop_home_dir, - # self.__params.zookeeper_quorum, - # "parser_error", - # num_partitions, replication_factor, - # retention_bytes)) Logger.info("Done creating Kafka topics") - def init_indexing_config(self): - Logger.info('Loading indexing config into ZooKeeper') - Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) - def start_indexing_topology(self): Logger.info("Starting Metron indexing topology: {}".format(self.__indexing)) - start_cmd_template = """{}/bin/start_indexing_topology.sh \ + start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \ -s {} \ -z {}""" Logger.info('Starting ' + self.__indexing) @@ -122,5 +101,35 @@ def stop_indexing_topology(self): def restart_indexing_topology(self): Logger.info('Restarting the indexing topologies') self.stop_indexing_topology() - self.start_indexing_topology() - Logger.info('Done restarting the indexing topologies') + + # Wait for old topology to be cleaned up by Storm, before starting again. + retries = 0 + topology_active = self.is_topology_active() + while topology_active and retries < 3: + Logger.info('Existing topology still active. Will wait and retry') + time.sleep(40) + topology_active = self.is_topology_active() + retries += 1 + + if not topology_active: + self.start_indexing_topology() + Logger.info('Done restarting the indexing topologies') + else: + Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + + def is_topology_active(self): + cmd_retrieve = "storm list | grep 'indexing'" + proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + Logger.info("Retrieval response is: %s" % stdout) + Logger.warning("Error response is: %s" % stderr) + + fields = stdout.split() + if len(fields) < 2: + Logger.warning("Indexing topology is not running") + return False + + # Get the second column, which is status. We already know first column is indexing) + status = stdout.split()[1] + running_status_set = {'ACTIVE', 'REBALANCING'} + return status in running_status_set diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py index 30eb671c56..e233379bb1 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py @@ -14,6 +14,7 @@ limitations under the License. """ +from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script @@ -34,7 +35,6 @@ def start(self, env, upgrade_type=None): env.set_params(params) commands = Commands(params) commands.init_kafka_topics() - commands.init_indexing_config() commands.start_indexing_topology() def stop(self, env, upgrade_type=None): @@ -46,7 +46,10 @@ def stop(self, env, upgrade_type=None): def status(self, env): import params env.set_params(params) - Logger.info('Status of the Master') + commands = Commands(params) + + if not commands.is_topology_active(): + raise ComponentIsNotRunning() def restart(self, env): import params @@ -60,12 +63,6 @@ def kafkabuild(self, env, upgrade_type=None): commands = Commands(params) commands.init_kafka_topics() - def zookeeperbuild(self, env, upgrade_type=None): - import params - env.set_params(params) - commands = Commands(params) - commands.init_indexing_config() - if __name__ == "__main__": Indexing().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py index cba46b602b..6440005faf 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py @@ -19,11 +19,11 @@ from resource_management.libraries.functions.expect import expect if OSCheck.is_windows_family(): - from params_windows import * + pass else: from params_linux import * java_home = config['hostLevelParams']['java_home'] java_version = expect("/hostLevelParams/java_version", int) -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) \ No newline at end of file +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py index faf78aaca6..ee5f546c25 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py @@ -18,33 +18,24 @@ import os from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import format from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.default import default from resource_management.libraries.script import Script -# server configurations +# Server configurations config = Script.get_config() hostname = config['hostname'] metron_home = config['configurations']['metron-indexing']['metron_home'] -metron_zookeeper_config_dir = config['configurations']['metron-indexing']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology'] yum_repo_type = 'local' -# hadoop params +# Hadoop params hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() -# Kafka -stack_root = Script.get_stack_root() -kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin") -metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention'] - -# zookeeper +# Zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) has_zk_host = not len(zk_hosts) == 0 zookeeper_quorum = None @@ -57,7 +48,19 @@ # last port config zookeeper_quorum += ':' + zookeeper_clientPort -# kafka -kafka_host = config['clusterHostInfo']['kafka_broker_hosts'][0] -# TODO -kafka_port = 6667 +# Kafka +stack_root = Script.get_stack_root() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") +metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention'] + +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py index fec2a30512..c02d695919 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py @@ -16,23 +16,22 @@ """ from __future__ import print_function -import sys - from resource_management.libraries.script import Script -from resource_management.core.resources.system import Execute -import subprocess + +from commands import Commands class ServiceCheck(Script): def service_check(self, env): - # import params - # env.set_params(params) - # - # doc = '{"name": "Ambari Smoke test"}' - # index = "ambari_smoke_test" - # - # print("Running parsers service check", file=sys.stdout) - exit(0) + import params + env.set_params(params) + + commands = Commands(params) + if commands.is_topology_active(): + exit(0) + else: + exit(1) + if __name__ == "__main__": ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json index 492f0a0bd8..ee1b225084 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json @@ -25,4 +25,4 @@ } ] } -} \ No newline at end of file +} diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py index 351250887a..a38c182f43 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py @@ -30,13 +30,13 @@ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) -class MetronIndexingServiceAdvisor(service_advisor.ServiceAdvisor): +class INDEXING020BETAServiceAdvisor(service_advisor.ServiceAdvisor): # colocate Metron Parser Master with KAFKA_BROKERs - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def colocateService(self, hostsComponentsMap, serviceComponents): indexingMasterComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "INDEXING_MASTER"] indexingMasterComponent = indexingMasterComponent[0] - if not stackAdvisor.isComponentHostsPopulated(indexingMasterComponent): + if not self.isComponentHostsPopulated(indexingMasterComponent): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "INDEXING_MASTER"} not in hostComponents: @@ -44,21 +44,19 @@ def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "INDEXING_MASTER"} in hostComponents: hostsComponentsMap[hostName].remove({"name": "INDEXING_MASTER"}) - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def getServiceComponentLayoutValidations(self, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] - hostsCount = len(hostsList) - masterHosts = self.getHosts(componentsList, "INDEXING_MASTER") - expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) + indexingHosts = self.getHosts(componentsList, "INDEXING_MASTER") + brokerHosts = self.getHosts(componentsList, "KAFKA_BROKER") items = [] - mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) + mismatchHosts = sorted(set(indexingHosts).symmetric_difference(set(brokerHosts))) if len(mismatchHosts) > 0: hostsString = ', '.join(mismatchHosts) - message = "Metron Parsers Master must be installed on Kafka Brokers. " \ + message = "Metron Indexing Master must be installed on Kafka Brokers. " \ "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( len(mismatchHosts), hostsString) items.append( diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml new file mode 100755 index 0000000000..e406780977 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml @@ -0,0 +1,26 @@ + + + + + 2.0 + + + INDEXING + 0.2.0BETA + common-services/INDEXING/0.2.0BETA + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json similarity index 67% rename from metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json rename to metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json index aff8bf1eb0..f58c59c322 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json @@ -3,6 +3,7 @@ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", "general_deps" : { "_comment" : "dependencies for all cases", - "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] + "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], + "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"] } -} \ No newline at end of file +} diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml deleted file mode 100755 index c7b407973e..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.5/services/INDEXING/metainfo.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - 2.0 - - - INDEXING - 0.2.0BETA - common-services/INDEXING/0.2.0BETA - - - \ No newline at end of file diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties index a1516d6aba..c2c10af059 100644 --- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties +++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties @@ -57,10 +57,13 @@ org.apache.metron.metrics.TelemetryParserBolt.fails=true bolt.hdfs.batch.size=5000 bolt.hdfs.field.delimiter=| +bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy +bolt.hdfs.rotation.policy.units=DAYS +bolt.hdfs.rotation.policy.count=1 + bolt.hdfs.file.rotation.size.in.mb=5 bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020 bolt.hdfs.wip.file.path=/paloalto/wip bolt.hdfs.finished.file.path=/paloalto/rotated bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec index.hdfs.output=/tmp/metron/enriched - From ae7a62af1ce17abc86897f26d05c4fb751095134 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Mon, 22 Aug 2016 11:40:48 -0400 Subject: [PATCH 17/82] remove errant file --- .../PARSERS/0.2.0BETA/service_advisor.pyc | Bin 3700 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc deleted file mode 100644 index 9f4735c2c1f3b7fb3eec8f58a9b6d9a631ccb277..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3700 zcmcgu&2k&Z5$**@kXn)=_2)!!;u!5>^N*N--2t^T8b|KMXm5eN~18~8` zF0?Zcg%WW{DLJPq50FdVA+M8L4!NdsOeJ6UE+N`EJ0N#ZNOX+Rp|C`;L&uop^Aa7GF6L!AE|V_NXo*J47n2I<6?*6J|MBVt z@e1ix3d{5k+uchuF>o(((9zxC|qXBA3vw#4bl#cu2Kk@m}6;$SDCm;14vt< z<1IRZiof~#DEXY0^ffwlDf=5a$JfcxWg6X}cSKION~4=B080J(apKU|AvD^e(JdOl zESG*y%=hB>1Gd+%vMbELN<{9HsIL>FLuFHCPqb=Gf<9h;I7qRrWS-a7pH<@ruA4S*3^{pwHDd!!%l0bsRmK3-7qrw zCLDbh*%L5CMxCYEsTx2|5QY&S8pJ9}Kp2Y(Zl;Gp7DmYshfZc$G(52?Jxg?EPNE4; z>aYd=-h~<_mnsLszF9gg)W2kD;edPUIi#?e4{F=03I_L1v2^F|FJ$XDn5iVSYHHXV zXv=bye$&?z3oW72coIiJ(${WbkyeEWsP+PfvxKx~126|-fl3GBnSxCP)^!zrC)Q5B ztk=)Z&T4@Kq?TsGdYn7g)StkOZNGURs=Ds8B-X~LOuw2&8NAt>sbB&{`auus#le|k ze{wt|$T<5fgKv}JJ!Oh0e=8b!fZ0PATEW!I;ewBXMBQols^#BNj~afjRJ<$CQj|VV)3L*o?m5U`>fVV@$BGj{()ggbY7y}mx|_PIxlmbyN!Zy8hB;*ZPH+A~=O|7negr9hqWLm^pD%;ALJ`zi=34gf zyh5iH%Kj`QoKlchm{jnON|zQP+)3y=0>FK6iKmcxun=FpPp~@3`8$)Ckk{_qWny2Yx8V6Q~!h5P`##X?F#MDVB(@c+nUr|$djE)DE(Raf8 z7t+i@FtOJ_EJIubRVW_@kyICGpPLPwUEdV$^gb`AB2;ae|sy_`TdAz6^#Pvf)? z&-gjUKLK5mj+_nWXU?W`!*QKYoe%K{gxU0EJObfuWB@}LKm%}hUv6E%7!YRIt`Q*1 z3`;=gPXwLIn1QWAzzK&?r`9;9)co^c(2yFFKYUe(-H zVJgkmpp2WrOjEJl0aBxiJ+85F{urr!;t!9b((_w#c{ zX_N?aevk3k`&GuZ$A!h?y`mOdZbYpjP*M&SYMCdvGVznMOnWiUxdL(6zFp3QSJqhQ zB91wk&zG}yaJRFT?}B5g7`b^j@Quc-`!Fnzq|iMsrBWf$YH=OwpXBVkU8MQ2JsEmG z#)Gc)9OVEp+h8^yVIb#Y=jYBR&U)#(^TSf5 z^r3SdMW=$Vit}kHFX$ZYu3)~~m0am|$7wi?b+_xWcf2n!ytnatHq_%p`zap7v96S^ ztW|%2iJ+j{4bwi9WJux$CD#-LoI8Wfg1+vd^9c+lfjyBOVU$U$WAq0-BlO%}zRec) z`a53hX{Wo}^1M4NiV@>?8aw+w(*!Xn`ptHyyVrWslmk*UHaUJUk#)ZR6kZc`p1$b% zhtIs7rpLD!AvNXZjBK_{(n6a6L$-@MwZ|3 Date: Thu, 25 Aug 2016 13:34:19 -0400 Subject: [PATCH 18/82] Working ES and Kibana install. --- .../ELASTICSEARCH/2.3.3/metainfo.xml | 29 + .../addon-services/KIBANA/4.5.1/metainfo.xml | 4 + .../2.3.3/configuration/elastic-env.xml | 58 + .../2.3.3/configuration/elastic-site.xml | 180 ++ .../2.3.3/configuration/elastic-sysconfig.xml | 98 ++ .../ELASTICSEARCH/2.3.3/metainfo.xml | 73 + .../2.3.3/package/scripts/elastic.py | 68 + .../2.3.3/package/scripts/elastic_master.py | 79 + .../2.3.3/package/scripts/elastic_slave.py | 76 + .../2.3.3/package/scripts/params.py | 77 + .../package/scripts/properties_config.py} | 29 +- .../2.3.3/package/scripts/service_check.py | 80 + .../2.3.3/package/scripts/slave.py | 65 + .../2.3.3/package/scripts/status_params.py | 26 + .../templates/elasticsearch.master.yaml.j2 | 84 + .../templates/elasticsearch.slave.yaml.j2 | 84 + .../2.3.3/role_command_order.json | 8 + .../package/scripts/dashboard/__init__.py | 16 + .../package/scripts/dashboard/dashboard.p | 1539 +++++++++++++++++ .../scripts/dashboard/dashboardindex.py | 95 + .../4.5.1/package/scripts/kibana_master.py | 1 - .../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 + .../src/main/resources/mpack.json | 18 + .../KIBANA/4.5.1/configuration/kibana-env.xml | 54 + .../4.5.1/configuration/kibana-site.xml | 112 ++ .../common-services/KIBANA/4.5.1/metainfo.xml | 75 + .../4.5.1/package/scripts/kibana_master.py | 146 ++ .../KIBANA/4.5.1/package/scripts/params.py | 47 + .../HDP/2.4/services/KIBANA/metainfo.xml | 29 + 29 files changed, 3256 insertions(+), 22 deletions(-) create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{KIBANA/4.5.1/package/scripts/kibana.py => ELASTICSEARCH/2.3.3/package/scripts/properties_config.py} (57%) create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml new file mode 100755 index 0000000000..e2e6cddb09 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml @@ -0,0 +1,29 @@ + + + + + 2.0 + + + ELASTICSEARCH + 2.3.3 + common-services/ELASTICSEARCH/2.3.3 + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml index 494f71b355..156a34ad1c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml @@ -20,6 +20,10 @@ 2.0 + + ELASTICSEARCH + common-services/ELASTICSEARCH/2.3.3 + KIBANA common-services/KIBANA/4.5.1 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml new file mode 100755 index 0000000000..2939c284a0 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml @@ -0,0 +1,58 @@ + + + + + + + elastic_user + elasticsearch + USER + The user for Elasticsearch + + + user_group + elasticsearch + The group for Elasticsearch + + + elastic_log_dir + /var/log/elasticsearch + Log directory for elastic + + + elastic_pid_dir + /var/run/elasticsearch + The directory for pid files + + + + + content + This is the jinja template for elastic-env.sh file + +#!/bin/bash + +# Set ELASTICSEARCH specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java64_home}} +export PATH=$PATH:$JAVA_HOME/bin + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml new file mode 100755 index 0000000000..fb3a44395d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -0,0 +1,180 @@ + + + + + + + + + cluster_name + metron + Cluster name identifies your cluster + + + zen_discovery_ping_unicast_hosts + + + Unicast discovery list of hosts to act as gossip routers, in comma separated format. + + + index_number_of_shards + 4 + Set the number of shards (splits) of an index + + + index_number_of_replicas + 2 + Set the number of replicas (additional copies) of an index + + + + path_data + "/opt/lmm/es_data" + Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second" + + + + transport_tcp_port + 9300-9400 + Set a custom port for the node to node communication + + + http_port + 9200-9300 + Set a custom port to listen for HTTP traffic + + + + discovery_zen_ping_multicast_enabled + false + master eligible nodes + + + discovery_zen_ping_timeout + 3s + Wait for ping responses for master discovery + + + discovery_zen_fd_ping_interval + 15s + Wait for ping for cluster discovery + + + discovery_zen_fd_ping_timeout + 60s + Wait for ping for cluster discovery + + + discovery_zen_fd_ping_retries + 5 + Number of ping retries before blacklisting + + + + gateway_recover_after_data_nodes + 3 + Recover as long as this many data or master nodes have joined the cluster. + + + recover_after_time + 15m + recover_after_time + + + expected_data_nodes + 0 + expected_data_nodes + + + + index_merge_scheduler_max_thread_count + 5 + index.merge.scheduler.max_thread_count + + + indices_memory_index_store_throttle_type + none + index_store_throttle_type + + + index_refresh_interval + 1s + index refresh interval + + + index_translog_flush_threshold_size + 5g + index_translog_flush_threshold_size + + + indices_memory_index_buffer_size + 10% + Percentage of heap used for write buffers + + + bootstrap_mlockall + true + The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out + + + threadpool_bulk_queue_size + 3000 + It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request + + + threadpool_index_queue_size + 1000 + It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request + + + indices_cluster_send_refresh_mapping + false + In order to make the index request more efficient, we have set this property on our data nodes + + + indices_fielddata_cache_size + 25% + You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index + + + cluster_routing_allocation_disk_watermark_high + 0.99 + Property used when multiple drives are used to understand max thresholds + + + cluster_routing_allocation_disk_threshold_enabled + true + Property used when multiple drives are used to understand if thresholding is active + + + cluster_routing_allocation_disk_watermark_low + .97 + Property used when multiple drives are used to understand min thresholds + + + cluster_routing_allocation_node_concurrent_recoveries + 4 + Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart + + + network_host + _lo_,_eth0_ + Network interface(s) will bind to. + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml new file mode 100755 index 0000000000..58e4916062 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml @@ -0,0 +1,98 @@ + + + + + + + elastic_home + /usr/share/elasticsearch/ + Elasticsearch Home Directory + + + data_dir + /var/lib/elasticsearch/ + Elasticsearch Data Directory + + + work_dir + /tmp/elasticsearch/ + Elasticsearch Work Directory + + + conf_dir + /etc/elasticsearch/ + Elasticsearch Configuration Directory + + + heap_size + 128m + Heap size + + + max_open_files + 65535 + Maximum number of open files + + + max_map_count + 262144 + Maximum number of memory map areas for process + + + + + content + This is the jinja template for elastic-env.sh file + +# Directory where the Elasticsearch binary distribution resides +ES_HOME={{elastic_home}} + +# Heap Size (defaults to 256m min, 1g max) +ES_HEAP_SIZE={{heap_size}} + +# Maximum number of open files +MAX_OPEN_FILES={{max_open_files}} + +# Maximum number of VMA (Virtual Memory Areas) a process can own +MAX_MAP_COUNT={{max_map_count}} + +# Elasticsearch log directory +LOG_DIR={{log_dir}} + +# Elasticsearch data directory +DATA_DIR={{data_dir}} + +# Elasticsearch work directory +WORK_DIR={{work_dir}} + +# Elasticsearch conf directory +CONF_DIR={{conf_dir}} + +# User to run as, change this to a specific elasticsearch user if possible +# Also make sure, this user can write into the log directories in case you change them +# This setting only works for the init script, but has to be configured separately for systemd startup +ES_USER={{elastic_user}} + +# Additional Java OPTS +ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled +-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps +-XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8" + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml new file mode 100755 index 0000000000..25f98e58e9 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml @@ -0,0 +1,73 @@ + + + + 2.0 + + + ELASTICSEARCH + Elasticsearch + Indexing and Search + 2.3.3 + + + ES_MASTER + Elasticsearch Master + MASTER + 1+ + + + PYTHON + 600 + + + + ES_SLAVE + Elasticsearch Data Node + SLAVE + 3+ + + + PYTHON + 600 + + + + + + any + + + elasticsearch-2.3.3 + + + + + + + PYTHON + 300 + + + elastic-env + elastic-site + elastic-sysconfig + + true + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py new file mode 100755 index 0000000000..bd858cd66d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template + + +def elastic(): + print "INSIDE THE %s" % __file__ + import params + + params.path_data = params.path_data.replace('"', '') + data_path = params.path_data.replace(' ', '').split(',') + data_path[:] = [x.replace('"', '') for x in data_path] + + directories = [params.log_dir, params.pid_dir, params.conf_dir] + directories = directories + data_path + + Directory(directories, + create_parents=True, + # recursive=True, + mode=0755, + owner=params.elastic_user, + group=params.elastic_user + ) + + print "Master env: ""{}/elastic-env.sh".format(params.conf_dir) + File("{}/elastic-env.sh".format(params.conf_dir), + owner=params.elastic_user, + content=InlineTemplate(params.elastic_env_sh_template) + ) + + configurations = params.config['configurations']['elastic-site'] + + print "Master yml: ""{}/elasticsearch.yml".format(params.conf_dir) + File("{}/elasticsearch.yml".format(params.conf_dir), + content=Template( + "elasticsearch.master.yaml.j2", + configurations=configurations), + owner=params.elastic_user, + group=params.elastic_user + ) + + print "Master sysconfig: /etc/sysconfig/elasticsearch" + File(format("/etc/sysconfig/elasticsearch"), + owner="root", + group="root", + content=InlineTemplate(params.sysconfig_template) + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py new file mode 100755 index 0000000000..5fc29cf32b --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py @@ -0,0 +1,79 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from elastic import elastic +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script + + +class Elasticsearch(Script): + def install(self, env): + import params + env.set_params(params) + + print 'Install the Master' + Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') + Execute("echo \"[elasticsearch-2.x]\n" + "name=Elasticsearch repository for 2.x packages\n" + "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" + "gpgcheck=1\n" + "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" + "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") + + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + + elastic() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + stop_cmd = format("service elasticsearch stop") + print 'Stop the Master' + Execute(stop_cmd) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + self.configure(env) + start_cmd = format("service elasticsearch start") + print 'Start the Master' + Execute(start_cmd) + + def status(self, env): + import params + env.set_params(params) + status_cmd = format("service elasticsearch status") + print 'Status of the Master' + Execute(status_cmd) + + def restart(self, env): + import params + env.set_params(params) + self.configure(env) + restart_cmd = format("service elasticsearch restart") + print 'Restarting the Master' + Execute(restart_cmd) + +if __name__ == "__main__": + Elasticsearch().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py new file mode 100755 index 0000000000..e65bd8f28e --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py @@ -0,0 +1,76 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute +from slave import slave + + +class Elasticsearch(Script): + def install(self, env): + import params + env.set_params(params) + print 'Install the Slave' + Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') + Execute("echo \"[elasticsearch-2.x]\n" + "name=Elasticsearch repository for 2.x packages\n" + "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" + "gpgcheck=1\n" + "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" + "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + slave() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + stop_cmd = format("service elasticsearch stop") + print 'Stop the Slave' + Execute(stop_cmd) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + start_cmd = format("service elasticsearch start") + print 'Start the Slave' + Execute(start_cmd) + + def status(self, env): + import params + env.set_params(params) + status_cmd = format("service elasticsearch status") + print 'Status of the Slave' + Execute(status_cmd) + + def restart(self, env): + import params + env.set_params(params) + self.configure(env) + restart_cmd = format("service elasticsearch restart") + print 'Restarting the Slave' + Execute(restart_cmd) + + +if __name__ == "__main__": + Elasticsearch().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py new file mode 100755 index 0000000000..c3e9169416 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] +data_dir = config['configurations']['elastic-sysconfig']['data_dir'] +work_dir = config['configurations']['elastic-sysconfig']['work_dir'] +conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] +heap_size = config['configurations']['elastic-sysconfig']['heap_size'] +max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] +max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] + +elastic_user = config['configurations']['elastic-env']['elastic_user'] +user_group = config['configurations']['elastic-env']['user_group'] +log_dir = config['configurations']['elastic-env']['elastic_log_dir'] +pid_dir = '/var/run/elasticsearch' +pid_file = '/var/run/elasticsearch/elasticsearch.pid' +hostname = config['hostname'] +java64_home = config['hostLevelParams']['java_home'] +elastic_env_sh_template = config['configurations']['elastic-env']['content'] +sysconfig_template = config['configurations']['elastic-sysconfig']['content'] + +cluster_name = config['configurations']['elastic-site']['cluster_name'] +zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] + +path_data = config['configurations']['elastic-site']['path_data'] +http_port = config['configurations']['elastic-site']['http_port'] +transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] + +recover_after_time = config['configurations']['elastic-site']['recover_after_time'] +gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] +expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] +discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled'] +index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] +index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] +index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] +indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] +index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] +index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] +indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] +bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall'] +threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] +cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] +cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] +cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'] +cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] +indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] +indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'] +threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] + +discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] +discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval'] +discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] +discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] + +network_host = config['configurations']['elastic-site']['network_host'] \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py similarity index 57% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py index 32ebf68631..ef9f6dd77d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py @@ -16,32 +16,19 @@ See the License for the specific language governing permissions and limitations under the License. -kibana service params. - """ -from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate -from resource_management.core.source import Template - - -def kibana(): - print "INSIDE THE %s" % __file__ - import params - directories = [params.log_dir, params.pid_dir, params.conf_dir] - Directory(directories, - # recursive=True, - mode=0755, - owner=params.kibana_user, - group=params.kibana_user - ) +def properties_inline_template(configurations): + return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }} +{% endfor %}''', configurations_dict=configurations) - print "Master env: ""{}/kibana.yml".format(params.conf_dir) - File("{}/kibana.yml".format(params.conf_dir), - owner=params.kibana_user, - content=InlineTemplate(params.kibana_yml_template) - ) +def properties_config(filename, configurations=None, conf_dir=None, + mode=None, owner=None, group=None, brokerid=None): + config_content = properties_inline_template(configurations) + File(format("{conf_dir}/{filename}"), content=config_content, owner=owner, + group=group, mode=mode) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py new file mode 100755 index 0000000000..9615d83405 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from __future__ import print_function + +import sys + +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute +import subprocess + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + doc = '{"name": "Ambari Smoke test"}' + index = "ambari_smoke_test" + + print("Running Elastic search service check", file=sys.stdout) + + # Make sure the service is actually up. We can live without everything allocated. + # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can + # also hit ES before things are green. + host = "localhost:9200" + Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host, + logoutput=True, + tries=6, + try_sleep=20 + ) + + # Put a document into a new index. + + Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) + + # Retrieve the document. Use subprocess because we actually need the results here. + cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index) + proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + response_retrieve = stdout + print("Retrieval response is: %s" % response_retrieve) + expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ + % (index, doc) + + # Delete the index + cmd_delete = "curl -XDELETE '%s/%s'" % (host, index) + proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + response_delete = stdout + print("Delete index response is: %s" % response_retrieve) + expected_delete = '{"acknowledged":true}' + + if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): + print("Smoke test able to communicate with Elasticsearch") + else: + print("Elasticsearch service unable to retrieve document.") + sys.exit(1) + + exit(0) + + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py new file mode 100755 index 0000000000..a134160e13 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template + + +def slave(): + import params + + params.path_data = params.path_data.replace('"', '') + data_path = params.path_data.replace(' ', '').split(',') + data_path[:] = [x.replace('"', '') for x in data_path] + + directories = [params.log_dir, params.pid_dir, params.conf_dir] + directories = directories + data_path + + Directory(directories, + create_parents=True, + mode=0755, + owner=params.elastic_user, + group=params.elastic_user, + cd_access="a" + ) + + File("{}/elastic-env.sh".format(params.conf_dir), + owner=params.elastic_user, + content=InlineTemplate(params.elastic_env_sh_template) + ) + + configurations = params.config['configurations']['elastic-site'] + + File("{}/elasticsearch.yml".format(params.conf_dir), + content=Template( + "elasticsearch.slave.yaml.j2", + configurations=configurations), + owner=params.elastic_user, + group=params.elastic_user + ) + + print "Master sysconfig: /etc/sysconfig/elasticsearch" + File(format("/etc/sysconfig/elasticsearch"), + owner="root", + group="root", + content=InlineTemplate(params.sysconfig_template) + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py new file mode 100755 index 0000000000..9cfb5cf014 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +config = Script.get_config() + +elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] +elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 new file mode 100755 index 0000000000..a9de018450 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 @@ -0,0 +1,84 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +cluster: + name: {{cluster_name}} + routing: + allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} + allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} + allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} + allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} + +discovery: + zen: + ping: + multicast: + enabled: {{discovery_zen_ping_multicast_enabled}} + unicast: + hosts: "{{zen_discovery_ping_unicast_hosts}}" + +node: + data: false + master: true + name: {{hostname}} +path: + data: {{path_data}} + +http.cors.enabled: true + +port: {{http_port}} + +transport: + tcp: + port: {{transport_tcp_port}} + +gateway: + recover_after_data_nodes: {{gateway_recover_after_data_nodes}} + recover_after_time: {{recover_after_time}} + expected_data_nodes: {{expected_data_nodes}} + +index: + number_of_shards: {{index_number_of_shards}} + merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} + translog.flush_threshold_size: {{index_translog_flush_threshold_size}} + refresh_interval: {{index_refresh_interval}} + number_of_replicas: {{index_number_of_replicas}} + +indices: + memory: + index_buffer_size: {{indices_memory_index_buffer_size}} + store.throttle.type: {{indices_memory_index_store_throttle_type}} + fielddata: + cache.size: {{indices_fielddata_cache_size}} + cluster: + send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} + +bootstrap.mlockall: {{bootstrap_mlockall}} + +threadpool: + bulk: + queue_size: {{threadpool_bulk_queue_size}} + index: + queue_size: {{threadpool_index_queue_size}} + +discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} +discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} +discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} +discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} + +network.host: {{network_host}} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 new file mode 100755 index 0000000000..e88fc5f0b0 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 @@ -0,0 +1,84 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +cluster: + name: {{cluster_name}} + routing: + allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} + allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} + allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} + allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} + +discovery: + zen: + ping: + multicast: + enabled: {{discovery_zen_ping_multicast_enabled}} + unicast: + hosts: "{{zen_discovery_ping_unicast_hosts}}" + +node: + data: true + master: false + name: {{hostname}} +path: + data: {{path_data}} + +http.cors.enabled: true + +port: {{http_port}} + +transport: + tcp: + port: {{transport_tcp_port}} + +gateway: + recover_after_data_nodes: {{gateway_recover_after_data_nodes}} + recover_after_time: {{recover_after_time}} + expected_data_nodes: {{expected_data_nodes}} + +index: + number_of_shards: {{index_number_of_shards}} + merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} + translog.flush_threshold_size: {{index_translog_flush_threshold_size}} + refresh_interval: {{index_refresh_interval}} + number_of_replicas: {{index_number_of_replicas}} + +indices: + memory: + index_buffer_size: {{indices_memory_index_buffer_size}} + store.throttle.type: {{indices_memory_index_store_throttle_type}} + fielddata: + cache.size: {{indices_fielddata_cache_size}} + cluster: + send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} + +bootstrap.mlockall: {{bootstrap_mlockall}} + +threadpool: + bulk: + queue_size: {{threadpool_bulk_queue_size}} + index: + queue_size: {{threadpool_index_queue_size}} + +discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} +discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} +discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} +discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} + +network.host: {{network_host}} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json new file mode 100755 index 0000000000..130d018de6 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json @@ -0,0 +1,8 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"] + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py new file mode 100755 index 0000000000..8d2bad81df --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p new file mode 100755 index 0000000000..8327eb8ad9 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p @@ -0,0 +1,1539 @@ +(lp1 +(dp2 +V_score +p3 +F1 +sV_type +p4 +Vvisualization +p5 +sV_id +p6 +VWeb-Request-Type +p7 +sV_source +p8 +(dp9 +VvisState +p10 +V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p11 +sVdescription +p12 +V +sVtitle +p13 +VWeb Request Type +p14 +sVuiStateJSON +p15 +V{} +p16 +sVversion +p17 +I1 +sVsavedSearchId +p18 +Vweb-search +p19 +sVkibanaSavedObjectMeta +p20 +(dp21 +VsearchSourceJSON +p22 +V{"filter":[]} +p23 +sssV_index +p24 +V.kibana +p25 +sa(dp26 +V_score +p27 +F1 +sV_type +p28 +Vvisualization +p29 +sV_id +p30 +VTop-Snort-Alerts-by-Source +p31 +sV_source +p32 +(dp33 +VvisState +p34 +V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} +p35 +sVdescription +p36 +V +sVtitle +p37 +VTop Snort Alerts by Source +p38 +sVuiStateJSON +p39 +V{} +p40 +sVversion +p41 +I1 +sVkibanaSavedObjectMeta +p42 +(dp43 +VsearchSourceJSON +p44 +V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p45 +sssV_index +p46 +V.kibana +p47 +sa(dp48 +V_score +p49 +F1 +sV_type +p50 +Vvisualization +p51 +sV_id +p52 +VWelcome +p53 +sV_source +p54 +(dp55 +VvisState +p56 +V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} +p57 +sVdescription +p58 +V +sVtitle +p59 +VWelcome to Apache Metron +p60 +sVuiStateJSON +p61 +V{} +p62 +sVversion +p63 +I1 +sVkibanaSavedObjectMeta +p64 +(dp65 +VsearchSourceJSON +p66 +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p67 +sssV_index +p68 +V.kibana +p69 +sa(dp70 +V_score +p71 +F1 +sV_type +p72 +Vsearch +p73 +sV_id +p74 +Vsnort-search +p75 +sV_source +p76 +(dp77 +Vsort +p78 +(lp79 +Vtimestamp +p80 +aVdesc +p81 +asVhits +p82 +I0 +sVdescription +p83 +V +sVtitle +p84 +VSnort Alerts +p85 +sVversion +p86 +I1 +sVkibanaSavedObjectMeta +p87 +(dp88 +VsearchSourceJSON +p89 +V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +p90 +ssVcolumns +p91 +(lp92 +Vmsg +p93 +aVsig_id +p94 +aVip_src_addr +p95 +aVip_src_port +p96 +aVip_dst_addr +p97 +aVip_dst_port +p98 +assV_index +p99 +V.kibana +p100 +sa(dp101 +V_score +p102 +F1 +sV_type +p103 +Vsearch +p104 +sV_id +p105 +Vyaf-search +p106 +sV_source +p107 +(dp108 +Vsort +p109 +(lp110 +Vtimestamp +p111 +aVdesc +p112 +asVhits +p113 +I0 +sVdescription +p114 +V +sVtitle +p115 +VYAF +p116 +sVversion +p117 +I1 +sVkibanaSavedObjectMeta +p118 +(dp119 +VsearchSourceJSON +p120 +V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} +p121 +ssVcolumns +p122 +(lp123 +Vip_src_addr +p124 +aVip_src_port +p125 +aVip_dst_addr +p126 +aVip_dst_port +p127 +aVprotocol +p128 +aVduration +p129 +aVpkt +p130 +assV_index +p131 +V.kibana +p132 +sa(dp133 +V_score +p134 +F1 +sV_type +p135 +Vconfig +p136 +sV_id +p137 +V4.5.1 +p138 +sV_source +p139 +(dp140 +VbuildNum +p141 +I9892 +sVdefaultIndex +p142 +Vbro* +p143 +ssV_index +p144 +V.kibana +p145 +sa(dp146 +V_score +p147 +F1 +sV_type +p148 +Vindex-pattern +p149 +sV_id +p150 +Vbro* +p151 +sV_source +p152 +(dp153 +Vfields +p154 +V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p155 +sVtimeFieldName +p156 +Vtimestamp +p157 +sVtitle +p158 +Vbro* +p159 +ssV_index +p160 +V.kibana +p161 +sa(dp162 +V_score +p163 +F1 +sV_type +p164 +Vvisualization +p165 +sV_id +p166 +VFlow-Duration +p167 +sV_source +p168 +(dp169 +VvisState +p170 +V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} +p171 +sVdescription +p172 +V +sVtitle +p173 +VFlow Duration +p174 +sVuiStateJSON +p175 +V{"vis":{"legendOpen":false}} +p176 +sVversion +p177 +I1 +sVkibanaSavedObjectMeta +p178 +(dp179 +VsearchSourceJSON +p180 +V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p181 +sssV_index +p182 +V.kibana +p183 +sa(dp184 +V_score +p185 +F1 +sV_type +p186 +Vvisualization +p187 +sV_id +p188 +VEvents +p189 +sV_source +p190 +(dp191 +VvisState +p192 +V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p193 +sVdescription +p194 +V +sVtitle +p195 +VEvents +p196 +sVuiStateJSON +p197 +V{"vis":{"legendOpen":false}} +p198 +sVversion +p199 +I1 +sVkibanaSavedObjectMeta +p200 +(dp201 +VsearchSourceJSON +p202 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p203 +sssV_index +p204 +V.kibana +p205 +sa(dp206 +V_score +p207 +F1 +sV_type +p208 +Vvisualization +p209 +sV_id +p210 +VWeb-Request-Header +p211 +sV_source +p212 +(dp213 +VvisState +p214 +V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} +p215 +sVdescription +p216 +V +sVtitle +p217 +VWeb Request Header +p218 +sVuiStateJSON +p219 +V{} +p220 +sVversion +p221 +I1 +sVkibanaSavedObjectMeta +p222 +(dp223 +VsearchSourceJSON +p224 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p225 +sssV_index +p226 +V.kibana +p227 +sa(dp228 +V_score +p229 +F1 +sV_type +p230 +Vvisualization +p231 +sV_id +p232 +VTop-Alerts-By-Host +p233 +sV_source +p234 +(dp235 +VvisState +p236 +V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} +p237 +sVdescription +p238 +V +sVtitle +p239 +VTop Alerts By Host +p240 +sVuiStateJSON +p241 +V{} +p242 +sVversion +p243 +I1 +sVsavedSearchId +p244 +Vsnort-search +p245 +sVkibanaSavedObjectMeta +p246 +(dp247 +VsearchSourceJSON +p248 +V{"filter":[]} +p249 +sssV_index +p250 +V.kibana +p251 +sa(dp252 +V_score +p253 +F1 +sV_type +p254 +Vvisualization +p255 +sV_id +p256 +VYAF-Flow(s) +p257 +sV_source +p258 +(dp259 +VvisState +p260 +V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p261 +sVdescription +p262 +V +sVtitle +p263 +VYAF Flows +p264 +sVuiStateJSON +p265 +V{} +p266 +sVversion +p267 +I1 +sVkibanaSavedObjectMeta +p268 +(dp269 +VsearchSourceJSON +p270 +V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p271 +sssV_index +p272 +V.kibana +p273 +sa(dp274 +V_score +p275 +F1 +sV_type +p276 +Vvisualization +p277 +sV_id +p278 +VTop-DNS-Query +p279 +sV_source +p280 +(dp281 +VvisState +p282 +V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} +p283 +sVdescription +p284 +V +sVtitle +p285 +VTop DNS Query +p286 +sVuiStateJSON +p287 +V{} +p288 +sVversion +p289 +I1 +sVkibanaSavedObjectMeta +p290 +(dp291 +VsearchSourceJSON +p292 +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p293 +sssV_index +p294 +V.kibana +p295 +sa(dp296 +V_score +p297 +F1 +sV_type +p298 +Vvisualization +p299 +sV_id +p300 +VTotal-Events +p301 +sV_source +p302 +(dp303 +VvisState +p304 +V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} +p305 +sVdescription +p306 +V +sVtitle +p307 +VEvent Count +p308 +sVuiStateJSON +p309 +V{} +p310 +sVversion +p311 +I1 +sVkibanaSavedObjectMeta +p312 +(dp313 +VsearchSourceJSON +p314 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p315 +sssV_index +p316 +V.kibana +p317 +sa(dp318 +V_score +p319 +F1 +sV_type +p320 +Vvisualization +p321 +sV_id +p322 +VEvent-Types +p323 +sV_source +p324 +(dp325 +VvisState +p326 +V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} +p327 +sVdescription +p328 +V +sVtitle +p329 +VEvent Sources +p330 +sVuiStateJSON +p331 +V{} +p332 +sVversion +p333 +I1 +sVkibanaSavedObjectMeta +p334 +(dp335 +VsearchSourceJSON +p336 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p337 +sssV_index +p338 +V.kibana +p339 +sa(dp340 +V_score +p341 +F1 +sV_type +p342 +Vvisualization +p343 +sV_id +p344 +VUnique-Location(s) +p345 +sV_source +p346 +(dp347 +VvisState +p348 +V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} +p349 +sVdescription +p350 +V +sVtitle +p351 +VGeo-IP Locations +p352 +sVuiStateJSON +p353 +V{} +p354 +sVversion +p355 +I1 +sVkibanaSavedObjectMeta +p356 +(dp357 +VsearchSourceJSON +p358 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p359 +sssV_index +p360 +V.kibana +p361 +sa(dp362 +V_score +p363 +F1 +sV_type +p364 +Vvisualization +p365 +sV_id +p366 +VSnort-Header +p367 +sV_source +p368 +(dp369 +VvisState +p370 +V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} +p371 +sVdescription +p372 +V +sVtitle +p373 +VSnort +p374 +sVuiStateJSON +p375 +V{} +p376 +sVversion +p377 +I1 +sVkibanaSavedObjectMeta +p378 +(dp379 +VsearchSourceJSON +p380 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p381 +sssV_index +p382 +V.kibana +p383 +sa(dp384 +V_score +p385 +F1 +sV_type +p386 +Vdashboard +p387 +sV_id +p388 +VMetron-Dashboard +p389 +sV_source +p390 +(dp391 +Vhits +p392 +I0 +sVtimeRestore +p393 +I00 +sVdescription +p394 +V +sVtitle +p395 +VMetron Dashboard +p396 +sVuiStateJSON +p397 +V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} +p398 +sVpanelsJSON +p399 +V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] +p400 +sVoptionsJSON +p401 +V{"darkTheme":false} +p402 +sVversion +p403 +I1 +sVkibanaSavedObjectMeta +p404 +(dp405 +VsearchSourceJSON +p406 +V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} +p407 +sssV_index +p408 +V.kibana +p409 +sa(dp410 +V_score +p411 +F1 +sV_type +p412 +Vvisualization +p413 +sV_id +p414 +VSnort-Alert-Types +p415 +sV_source +p416 +(dp417 +VvisState +p418 +V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} +p419 +sVdescription +p420 +V +sVtitle +p421 +VSnort Alert Types +p422 +sVuiStateJSON +p423 +V{} +p424 +sVversion +p425 +I1 +sVkibanaSavedObjectMeta +p426 +(dp427 +VsearchSourceJSON +p428 +V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p429 +sssV_index +p430 +V.kibana +p431 +sa(dp432 +V_score +p433 +F1 +sV_type +p434 +Vvisualization +p435 +sV_id +p436 +VFrequent-DNS-Queries +p437 +sV_source +p438 +(dp439 +VvisState +p440 +V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p441 +sVdescription +p442 +V +sVtitle +p443 +VFrequent DNS Requests +p444 +sVuiStateJSON +p445 +V{} +p446 +sVversion +p447 +I1 +sVkibanaSavedObjectMeta +p448 +(dp449 +VsearchSourceJSON +p450 +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p451 +sssV_index +p452 +V.kibana +p453 +sa(dp454 +V_score +p455 +F1 +sV_type +p456 +Vvisualization +p457 +sV_id +p458 +VLocation-Header +p459 +sV_source +p460 +(dp461 +VvisState +p462 +V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} +p463 +sVdescription +p464 +V +sVtitle +p465 +VEnrichment +p466 +sVuiStateJSON +p467 +V{} +p468 +sVversion +p469 +I1 +sVkibanaSavedObjectMeta +p470 +(dp471 +VsearchSourceJSON +p472 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p473 +sssV_index +p474 +V.kibana +p475 +sa(dp476 +V_score +p477 +F1 +sV_type +p478 +Vsearch +p479 +sV_id +p480 +Vweb-search +p481 +sV_source +p482 +(dp483 +Vsort +p484 +(lp485 +Vtimestamp +p486 +aVdesc +p487 +asVhits +p488 +I0 +sVdescription +p489 +V +sVtitle +p490 +VWeb Requests +p491 +sVversion +p492 +I1 +sVkibanaSavedObjectMeta +p493 +(dp494 +VsearchSourceJSON +p495 +V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +p496 +ssVcolumns +p497 +(lp498 +Vmethod +p499 +aVhost +p500 +aVuri +p501 +aVreferrer +p502 +aVip_src_addr +p503 +aVip_dst_addr +p504 +assV_index +p505 +V.kibana +p506 +sa(dp507 +V_score +p508 +F1 +sV_type +p509 +Vindex-pattern +p510 +sV_id +p511 +Vsnort* +p512 +sV_source +p513 +(dp514 +Vfields +p515 +V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p516 +sVtimeFieldName +p517 +Vtimestamp +p518 +sVtitle +p519 +Vsnort* +p520 +ssV_index +p521 +V.kibana +p522 +sa(dp523 +V_score +p524 +F1 +sV_type +p525 +Vindex-pattern +p526 +sV_id +p527 +Vyaf* +p528 +sV_source +p529 +(dp530 +Vfields +p531 +V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p532 +sVtimeFieldName +p533 +Vtimestamp +p534 +sVtitle +p535 +Vyaf* +p536 +ssV_index +p537 +V.kibana +p538 +sa(dp539 +V_score +p540 +F1 +sV_type +p541 +Vvisualization +p542 +sV_id +p543 +VDNS-Request(s) +p544 +sV_source +p545 +(dp546 +VvisState +p547 +V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p548 +sVdescription +p549 +V +sVtitle +p550 +VDNS Requests +p551 +sVuiStateJSON +p552 +V{} +p553 +sVversion +p554 +I1 +sVsavedSearchId +p555 +Vdns-search +p556 +sVkibanaSavedObjectMeta +p557 +(dp558 +VsearchSourceJSON +p559 +V{"filter":[]} +p560 +sssV_index +p561 +V.kibana +p562 +sa(dp563 +V_score +p564 +F1 +sV_type +p565 +Vvisualization +p566 +sV_id +p567 +VHTTP(S)-Requests +p568 +sV_source +p569 +(dp570 +VvisState +p571 +V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p572 +sVdescription +p573 +V +sVtitle +p574 +VWeb Requests +p575 +sVuiStateJSON +p576 +V{} +p577 +sVversion +p578 +I1 +sVsavedSearchId +p579 +Vweb-search +p580 +sVkibanaSavedObjectMeta +p581 +(dp582 +VsearchSourceJSON +p583 +V{"filter":[]} +p584 +sssV_index +p585 +V.kibana +p586 +sa(dp587 +V_score +p588 +F1 +sV_type +p589 +Vsearch +p590 +sV_id +p591 +Vdns-search +p592 +sV_source +p593 +(dp594 +Vsort +p595 +(lp596 +Vtimestamp +p597 +aVdesc +p598 +asVhits +p599 +I0 +sVdescription +p600 +V +sVtitle +p601 +VDNS Requests +p602 +sVversion +p603 +I1 +sVkibanaSavedObjectMeta +p604 +(dp605 +VsearchSourceJSON +p606 +V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +p607 +ssVcolumns +p608 +(lp609 +Vquery +p610 +aVqtype_name +p611 +aVanswers +p612 +aVip_src_addr +p613 +aVip_dst_addr +p614 +assV_index +p615 +V.kibana +p616 +sa(dp617 +V_score +p618 +F1 +sV_type +p619 +Vvisualization +p620 +sV_id +p621 +VFlow-Locations +p622 +sV_source +p623 +(dp624 +VvisState +p625 +V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} +p626 +sVdescription +p627 +V +sVtitle +p628 +VFlow Locations +p629 +sVuiStateJSON +p630 +V{} +p631 +sVversion +p632 +I1 +sVkibanaSavedObjectMeta +p633 +(dp634 +VsearchSourceJSON +p635 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p636 +sssV_index +p637 +V.kibana +p638 +sa(dp639 +V_score +p640 +F1 +sV_type +p641 +Vvisualization +p642 +sV_id +p643 +VUnusual-Referrers +p644 +sV_source +p645 +(dp646 +VvisState +p647 +V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} +p648 +sVdescription +p649 +V +sVtitle +p650 +VUnusual Referrers +p651 +sVuiStateJSON +p652 +V{} +p653 +sVversion +p654 +I1 +sVsavedSearchId +p655 +Vweb-search +p656 +sVkibanaSavedObjectMeta +p657 +(dp658 +VsearchSourceJSON +p659 +V{"filter":[]} +p660 +sssV_index +p661 +V.kibana +p662 +sa(dp663 +V_score +p664 +F1 +sV_type +p665 +Vvisualization +p666 +sV_id +p667 +VFrequent-DNS-Requests +p668 +sV_source +p669 +(dp670 +VvisState +p671 +V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} +p672 +sVdescription +p673 +V +sVtitle +p674 +VFrequent DNS Requests +p675 +sVuiStateJSON +p676 +V{} +p677 +sVversion +p678 +I1 +sVkibanaSavedObjectMeta +p679 +(dp680 +VsearchSourceJSON +p681 +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p682 +sssV_index +p683 +V.kibana +p684 +sa(dp685 +V_score +p686 +F1 +sV_type +p687 +Vvisualization +p688 +sV_id +p689 +VCountry +p690 +sV_source +p691 +(dp692 +VvisState +p693 +V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p694 +sVdescription +p695 +V +sVtitle +p696 +VBy Country +p697 +sVuiStateJSON +p698 +V{} +p699 +sVversion +p700 +I1 +sVkibanaSavedObjectMeta +p701 +(dp702 +VsearchSourceJSON +p703 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p704 +sssV_index +p705 +V.kibana +p706 +sa(dp707 +V_score +p708 +F1 +sV_type +p709 +Vvisualization +p710 +sV_id +p711 +VTop-Destinations +p712 +sV_source +p713 +(dp714 +VvisState +p715 +V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} +p716 +sVdescription +p717 +V +sVtitle +p718 +VTop Destinations +p719 +sVuiStateJSON +p720 +V{} +p721 +sVversion +p722 +I1 +sVkibanaSavedObjectMeta +p723 +(dp724 +VsearchSourceJSON +p725 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p726 +sssV_index +p727 +V.kibana +p728 +sa(dp729 +V_score +p730 +F1 +sV_type +p731 +Vvisualization +p732 +sV_id +p733 +VDNS-Requests-Header +p734 +sV_source +p735 +(dp736 +VvisState +p737 +V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"} +p738 +sVdescription +p739 +V +sVtitle +p740 +VDNS Requests +p741 +sVuiStateJSON +p742 +V{} +p743 +sVversion +p744 +I1 +sVkibanaSavedObjectMeta +p745 +(dp746 +VsearchSourceJSON +p747 +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p748 +sssV_index +p749 +V.kibana +p750 +sa(dp751 +V_score +p752 +F1 +sV_type +p753 +Vvisualization +p754 +sV_id +p755 +VYAF-Flows-Header +p756 +sV_source +p757 +(dp758 +VvisState +p759 +V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} +p760 +sVdescription +p761 +V +sVtitle +p762 +VYAF +p763 +sVuiStateJSON +p764 +V{} +p765 +sVversion +p766 +I1 +sVkibanaSavedObjectMeta +p767 +(dp768 +VsearchSourceJSON +p769 +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p770 +sssV_index +p771 +V.kibana +p772 +sa. \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py new file mode 100755 index 0000000000..f0903acf47 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +import cPickle as pickle +import argparse, sys, os.path +import errno +import os + + +class DashboardIndex(object): + + def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): + """ + :arg host: hostname of the node (default: localhost) + :arg port: port to use (integer, default: 9200) + :arg url_prefix: optional url prefix for elasticsearch + :arg timeout: default timeout in seconds (float, default: 10) + """ + self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) + + def get(self): + """ + Get .kibana index from Elasticsearch + """ + dotkibana = self.es.search(index='.kibana', size = 100) + return dotkibana['hits']['hits'] + + def load(self,filespec): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + data=[] + with open(filespec,'rb') as fp: + data = pickle.load(fp) + return data + + def save(self,filename,data): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + with open(filename,'wb') as fp: + pickle.dump(data,fp) + + def put(self,data): + """ + Bulk write data to Elasticsearch + :args data: data to be written (note: index name is specified in data) + """ + bulk(self.es,data) + + def main(self,args): + + if args.save: + print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) + self.save(filename=args.filespec,data=di.get()) + else: + """ + Loads Kibana Dashboard definition from disk and replaces .kibana on index + :args filespec: path/filename for saved file + """ + if not os.path.isfile(args.filespec): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) + self.es.indices.delete(index='.kibana', ignore=[400, 404]) + self.put(data=di.load(filespec=args.filespec)) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument("hostname", help="ES Hostname or IP", type=str) + parser.add_argument("port", help="ES Port", type=int) + parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) + parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") + args = parser.parse_args() + di = DashboardIndex(host=args.hostname,port=args.port) + di.main(args) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py index 7913c8cbd8..cfa1f39453 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -51,7 +51,6 @@ def install(self, env): "enabled=1\" > /etc/yum.repos.d/kibana.repo") self.install_packages(env) - self.loadtemplate(env) def configure(self, env): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json new file mode 100755 index 0000000000..448e10245d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "metron_ui", + "label": "Metron UI", + "requires_user_name": "false", + "component_name": "KIBANA_MASTER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "kibana_server_port", + "http_default_port": "5601", + "https_property": "kibana_server_port", + "https_default_port": "5601", + "regex": "^(\\d+)$", + "site": "kibana-env" + } + } + ] + } +} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json index a3ba3795ce..2fbb358697 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json @@ -40,6 +40,24 @@ "stack_version" : "2.5" } ] + }, + { + "service_name" : "ELASTICSEARCH", + "service_version" : "2.3.3", + "applicable_stacks" : [ + { + "stack_name" : "HDP", + "stack_version" : "2.3" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.4" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.5" + } + ] } ] } diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml new file mode 100755 index 0000000000..972945dfba --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml @@ -0,0 +1,54 @@ + + + + + + kibana_user + kibana + USER + + + + user_group + kibana + GROUP + + + + kibana_log_dir + /var/log/kibana + Log directory for Kibana + + + kibana_pid_dir + /var/run/kibana + PID directory for Kibana + + + kibana_es_url + + The Elasticsearch instance to use for all your queries. (http://eshost:9200) + + + kibana_server_port + 5000 + Kibana back end server port to use. + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml new file mode 100755 index 0000000000..c546e2c2ba --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml @@ -0,0 +1,112 @@ + + + + + + + content + kibana.yml template + This is the jinja template for kibana.yml file + +# Kibana is served by a back end server. This controls which port to use. +server.port: {{ kibana_port }} + +# The host to bind the server to. +# server.host: "0.0.0.0" + +# If you are running kibana behind a proxy, and want to mount it at a path, +# specify that path here. The basePath can't end in a slash. +# server.basePath: "" + +# The maximum payload size in bytes on incoming server requests. +# server.maxPayloadBytes: 1048576 + +# The Elasticsearch instance to use for all your queries. +elasticsearch.url: {{ es_url }} + +# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, +# then the host you use to connect to *this* Kibana instance will be sent. +# elasticsearch.preserveHost: true + +# Kibana uses an index in Elasticsearch to store saved searches, visualizations +# and dashboards. It will create a new index if it doesn't already exist. +# kibana.index: ".kibana" + +# The default application to load. +# kibana.defaultAppId: "discover" + +# If your Elasticsearch is protected with basic auth, these are the user credentials +# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana +# users will still need to authenticate with Elasticsearch (which is proxied through +# the Kibana server) +# elasticsearch.username: "user" +# elasticsearch.password: "pass" + +# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) +# server.ssl.cert: /path/to/your/server.crt +# server.ssl.key: /path/to/your/server.key + +# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) +# elasticsearch.ssl.cert: /path/to/your/client.crt +# elasticsearch.ssl.key: /path/to/your/client.key + +# If you need to provide a CA certificate for your Elasticsearch instance, put +# the path of the pem file here. +# elasticsearch.ssl.ca: /path/to/your/CA.pem + +# Set to false to have a complete disregard for the validity of the SSL +# certificate. +# elasticsearch.ssl.verify: true + +# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to +# request_timeout setting +# elasticsearch.pingTimeout: 1500 + +# Time in milliseconds to wait for responses from the back end or elasticsearch. +# This must be > 0 +# elasticsearch.requestTimeout: 30000 + +# Time in milliseconds for Elasticsearch to wait for responses from shards. +# Set to 0 to disable. +# elasticsearch.shardTimeout: 0 + +# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying +# elasticsearch.startupTimeout: 5000 + +# Set the path to where you would like the process id file to be created. +# pid.file: /var/run/kibana.pid + +# If you would like to send the log output to a file you can set the path below. +logging.dest: {{ log_dir }}/kibana.log + +# Set this to true to suppress all logging output. +# logging.silent: false + +# Set this to true to suppress all logging output except for error messages. +# logging.quiet: false + +# Set this to true to log all events, including system usage information and all requests. +# logging.verbose: false + + + content + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml new file mode 100755 index 0000000000..d14afbf5e8 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml @@ -0,0 +1,75 @@ + + + + 2.0 + + + KIBANA + Kibana + Kibana Dashboard + 4.5.1 + + + KIBANA_MASTER + Kibana Server + MASTER + 1 + + + PYTHON + 600 + + + + LOADTEMPLATE + false + + + PYTHON + + + + + + + + any + + + python-elasticsearch + + + kibana-4.5.1 + + + + + + kibana-env + kibana-site + + true + + + quicklinks.json + true + + + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py new file mode 100755 index 0000000000..7913c8cbd8 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -0,0 +1,146 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +kibana_master + +""" + + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script +from resource_management.libraries.functions.format import format +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate + +import errno +import os + +class Kibana(Script): + + def install(self, env): + + import params + env.set_params(params) + + Logger.info("Install Kibana Master") + + #TODO: Figure this out for all supported OSes + Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') + Execute("echo \"[kibana-4.x]\n" + "name=Kibana repository for 4.5.x packages\n" + "baseurl=http://packages.elastic.co/kibana/4.5/centos\n" + "gpgcheck=1\n" + "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" + "enabled=1\" > /etc/yum.repos.d/kibana.repo") + + self.install_packages(env) + self.loadtemplate(env) + + def configure(self, env): + + import params + env.set_params(params) + + Logger.info("Configure Kibana for Metron") + + directories = [params.log_dir, params.pid_dir, params.conf_dir] + Directory(directories, + # recursive=True, + mode=0755, + owner=params.kibana_user, + group=params.kibana_user + ) + + File("{}/kibana.yml".format(params.conf_dir), + owner=params.kibana_user, + content=InlineTemplate(params.kibana_yml_template) + ) + + def stop(self, env): + + import params + env.set_params(params) + + Logger.info("Stop Kibana Master") + + Execute("service kibana stop") + + def start(self, env): + + import params + env.set_params(params) + + self.configure(env) + + Logger.info("Start the Master") + + Execute("service kibana start") + + def restart(self,env): + + import params + env.set_params(params) + + self.configure(env) + + Logger.info("Restarting the Master") + + Execute("service kibana restart") + + def status(self, env): + + import params + env.set_params(params) + + Logger.info("Status of the Master") + + Execute("service kibana status") + + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) + def loadtemplate(self,env): + + from dashboard.dashboardindex import DashboardIndex + + import params + env.set_params(params) + + hostname = format("{es_host}") + port = int(format("{es_port}")) + + Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname,port)) + di = DashboardIndex(host=hostname,port=port) + + #Loads Kibana Dashboard definition from disk and replaces .kibana on index + templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)),'dashboard','dashboard.p') + if not os.path.isfile(templateFile): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), templateFile) + + Logger.info("Deleting .kibana index from Elasticsearch") + + di.es.indices.delete(index='.kibana', ignore=[400, 404]) + + Logger.info("Loading .kibana index from %s" % templateFile) + + di.put(data=di.load(filespec=templateFile)) + + +if __name__ == "__main__": + Kibana().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py new file mode 100755 index 0000000000..03fae0b759 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Kibana Params configurations + +""" + + +from resource_management.libraries.script import Script +from resource_management.libraries.functions import format +from urlparse import urlparse + +# server configurations +config = Script.get_config() + +kibana_home = '/usr/share/kibana/' +kibana_bin = '/usr/share/kibana/bin/' + +conf_dir = "/opt/kibana/config/" +kibana_user = config['configurations']['kibana-env']['kibana_user'] +user_group = config['configurations']['kibana-env']['user_group'] +log_dir = config['configurations']['kibana-env']['kibana_log_dir'] +pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] +pid_file = format("{pid_dir}/kibanasearch.pid") +es_url = config['configurations']['kibana-env']['kibana_es_url'] +parsed = urlparse(es_url) +es_host = parsed.netloc.split(':')[0] +es_port = parsed.netloc.split(':')[1] +kibana_port = config['configurations']['kibana-env']['kibana_server_port'] +hostname = config['hostname'] +java64_home = config['hostLevelParams']['java_home'] +kibana_yml_template = config['configurations']['kibana-site']['content'] diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml new file mode 100755 index 0000000000..494f71b355 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml @@ -0,0 +1,29 @@ + + + + 2.0 + + + KIBANA + common-services/KIBANA/4.5.1 + + + + From 32f7c63d17b0d62012d0e542cddfb9e81b0606c1 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Thu, 25 Aug 2016 14:19:50 -0400 Subject: [PATCH 19/82] Working parser integration --- .../PARSERS/0.2.0BETA/metainfo.xml | 27 ++ .../PARSERS/0.2.0BETA/role_command_order.json | 17 ++ .../configuration/metron-parsers.xml | 57 +++++ .../PARSERS/0.2.0BETA/metainfo.xml | 108 ++++++++ .../0.2.0BETA/package/scripts/commands.py | 235 ++++++++++++++++++ .../package/scripts/params/__init__.py | 18 ++ .../package/scripts/params/params.py | 32 +++ .../package/scripts/params/params_linux.py | 114 +++++++++ .../package/scripts/params/params_windows.py | 20 ++ .../package/scripts/params/status_params.py | 22 ++ .../package/scripts/parser_master.py | 77 ++++++ .../package/scripts/service_check.py | 40 +++ .../package/templates/metron-global.json | 1 + .../0.2.0BETA/quicklinks/quicklinks.json | 28 +++ .../PARSERS/0.2.0BETA/service_advisor.py | 68 +++++ 15 files changed, 864 insertions(+) create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..e0292cfabf --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml @@ -0,0 +1,27 @@ + + + + 2.0 + + + PARSERS + 0.2.0BETA + common-services/PARSERS/0.2.0BETA + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json new file mode 100755 index 0000000000..c8616469b6 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json @@ -0,0 +1,17 @@ +{ + "_comment": "Record format:", + "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps": { + "_comment": "dependencies for Metron Parsers", + "PARSER_MASTER-START": [ + "NAMENODE-START", + "ZOOKEEPER_SERVER-START", + "KAFKA_BROKER-START", + "STORM_REST_API-START" + ], + "_comment": "_SERVICE_CHECK-SERVICE_CHECK", + "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [ + "PARSER_MASTER-START" + ] + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml new file mode 100755 index 0000000000..71966eab9f --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml @@ -0,0 +1,57 @@ + + + + + + metron_home + /usr/metron/0.2.0BETA + Metron home directory + Metron home + + + parsers + bro,snort,yaf + Metron parsers to deploy + Metron parsers + + + metron_apps_hdfs_dir + /apps/metron + Metron apps HDFS dir + Metron apps HDFS dir + + + metron_zookeeper_config_dir + config/zookeeper + Metron Zookeeper config dir. Relative path to Metron home. + Metron Zookeeper config dir + + + metron_user + metron + USER + The user for Metron + Metron User + + + metron_group + metron + The group for Metron + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..f1e5ab7d90 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -0,0 +1,108 @@ + + + + 2.0 + + + PARSERS + Metron Parsers + Parsers + 0.2.0BETA + + + + PARSER_MASTER + Metron Parser Topologies + MASTER + 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + KAFKA/KAFKA_BROKER + cluster + + true + + + + + + PYTHON + 600 + + + metron-parsers + + + + + + + any + + + metron-common + + + metron-parsers + + + + + + + + PYTHON + 300 + + + + HDFS + KAFKA + STORM + ZOOKEEPER + + + + metron-parsers + + + + + quicklinks.json + true + + + + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py new file mode 100755 index 0000000000..6b33e55a55 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import re +import subprocess +import time + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import Template +from resource_management.libraries.functions import format + + +# Wrap major operations and functionality in this class +class Commands: + __params = None + __parser_list = None + __configured = False + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__parser_list = self.__get_parsers(params) + self.__configured = os.path.isfile(self.__params.configured_flag_file) + + # get list of parsers + def __get_parsers(self, params): + return params.parsers.replace(' ', '').split(',') + + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + # Possible storm topology status states + # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html + class StormStatus: + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + KILLED = "KILLED" + REBALANCING = "REBALANCING" + + def init_parsers(self): + Logger.info( + "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, + self.__params.metron_apps_dir)) + self.__params.HdfsResource(self.__params.metron_apps_dir, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + mode=0775, + source=self.__params.local_grok_patterns_dir) + + Logger.info("Creating global.json file") + File(self.__params.metron_zookeeper_config_path + '/global.json', + content=Template("metron-global.json"), + owner=self.__params.metron_user, + mode=0775) + Logger.info("Done initializing parser configuration") + + def get_parser_list(self): + return self.__parser_list + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + command_template = """{}/kafka-topics.sh \ + --zookeeper {} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""" + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = 10 + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating main topics for parsers") + for parser_name in self.get_parser_list(): + Logger.info("Creating topic'{}'".format(parser_name)) + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + parser_name, + num_partitions, + replication_factor, + retention_bytes)) + Logger.info("Creating topics for error handling") + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + "parser_invalid", + num_partitions, + replication_factor, + retention_bytes)) + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + "parser_error", + num_partitions, replication_factor, + retention_bytes)) + Logger.info("Done creating Kafka topics") + + def init_parser_config(self): + Logger.info('Loading parser config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) + + def start_parser_topologies(self): + Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) + start_cmd_template = """{}/bin/start_parser_topology.sh \ + -k {} \ + -z {} \ + -s {}""" + for parser in self.get_parser_list(): + Logger.info('Starting ' + parser) + Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, + self.__params.zookeeper_quorum, parser)) + + Logger.info('Finished starting parser topologies') + + def stop_parser_topologies(self): + Logger.info('Stopping parsers') + for parser in self.get_parser_list(): + Logger.info('Stopping ' + parser) + stop_cmd = 'storm kill ' + parser + Execute(stop_cmd) + Logger.info('Done stopping parser topologies') + + def restart_parser_topologies(self): + Logger.info('Restarting the parser topologies') + self.stop_parser_topologies() + attempt_count = 0 + while self.topologies_exist(): + if attempt_count > 2: + raise Exception("Unable to kill topologies") + attempt_count += 1 + time.sleep(10) + self.start_parser_topologies() + Logger.info('Done restarting the parser topologies') + + def topologies_exist(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + if stdout_lines: + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + return True + return False + + def topologies_running(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + all_running = False + if stdout_lines: + all_running = True + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + parser_found = False + is_running = False + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + status = items[1] + parser_found = True + is_running = self.__is_running(status) + all_running &= parser_found and is_running + return all_running + + def __get_status_lines(self, lines): + status_lines = [] + do_stat = False + skipped = 0 + for line in lines: + if line.startswith("Topology_name"): + do_stat = True + if do_stat and skipped == 2: + status_lines += [line] + elif do_stat: + skipped += 1 + return status_lines + + def __is_running(self, status): + return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py new file mode 100755 index 0000000000..242460e266 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py new file mode 100755 index 0000000000..953435de13 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.expect import expect + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +java_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py new file mode 100755 index 0000000000..de7dfc9ff2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import functools +import os + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +hostname = config['hostname'] +#print(config['configurations']) +metron_home = config['configurations']['metron-parsers']['metron_home'] +parsers = config['configurations']['metron-parsers']['parsers'] +metron_user = config['configurations']['metron-parsers']['metron_user'] +metron_group = config['configurations']['metron-parsers']['metron_group'] +metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] +#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') +configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +yum_repo_type = 'local' + +# hadoop params +stack_root = Script.get_stack_root() +hadoop_home_dir = stack_select.get_hadoop_dir("home") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") + +# zookeeper +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort + +# Kafka +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + +metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] + +local_grok_patterns_dir = format("{metron_home}/patterns") +hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") + +# for create_hdfs_directory +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") + +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + hadoop_bin_dir=hadoop_bin_dir, + hadoop_conf_dir=hadoop_conf_dir, + principal_name=hdfs_principal_name, + hdfs_site=hdfs_site, + default_fs=default_fs, + immutable_paths=get_not_managed_resources(), + dfs_type=dfs_type +) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py new file mode 100755 index 0000000000..4d11b358c9 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py @@ -0,0 +1,20 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +raise NotImplementedError diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py new file mode 100755 index 0000000000..ee83414f0c --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +parsers = "bro,yaf,snort" +configured_flag_file = "" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py new file mode 100755 index 0000000000..4a5b910918 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py @@ -0,0 +1,77 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.libraries.script import Script + +from commands import Commands + + +class ParserMaster(Script): + def get_component_name(self): + # TODO add this at some point - currently will cause problems with hdp-select + # return "parser-master" + pass + + def install(self, env): + from params import params + env.set_params(params) + commands = Commands(params) + commands.setup_repo() + Logger.info('Install RPM packages') + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) + + def start(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + if not commands.is_configured(): + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() + commands.set_configured() + commands.start_parser_topologies() + + def stop(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.stop_parser_topologies() + + def status(self, env): + from params import status_params + env.set_params(status_params) + commands = Commands(status_params) + if not commands.topologies_running(): + raise ComponentIsNotRunning() + + def restart(self, env): + from params import params + env.set_params(params) + commands = Commands(params) + commands.restart_parser_topologies() + + +if __name__ == "__main__": + ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py new file mode 100755 index 0000000000..6ef7e4aa14 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from __future__ import print_function + +from resource_management.libraries.script import Script + +from commands import Commands + + +class ServiceCheck(Script): + def service_check(self, env): + from params import params + env.set_params(params) + commands = Commands(params) + all_found = commands.topologies_running() + if all_found: + exit(0) + else: + exit(1) + + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json new file mode 100755 index 0000000000..0967ef424b --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json @@ -0,0 +1 @@ +{} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json new file mode 100755 index 0000000000..ee1b225084 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py new file mode 100755 index 0000000000..84b466a353 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -0,0 +1,68 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) + + +class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): + # colocate Metron Parser Master with KAFKA_BROKERs + def TODO_colocateService(self, hostsComponentsMap, serviceComponents): + parsersMasterComponent = [component for component in serviceComponents if + component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0] + if not self.isComponentHostsPopulated(parsersMasterComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: + hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"}) + if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) + + def TODO_getServiceComponentLayoutValidations(self, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + masterHosts = self.getHosts(componentsList, "PARSER_MASTER") + expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) + + items = [] + + mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Metron Parsers Master must be installed on Kafka Brokers. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( + len(mismatchHosts), hostsString) + items.append( + {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'}) + + return items From 8ab5c7b7d73cb6f98bb6957bb53e4ebfaaeab04d Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 5 Aug 2016 09:23:12 -0400 Subject: [PATCH 20/82] sample --- .../0.2.0BETA/configuration/metron.xml | 31 +++++++ .../PARSERS/0.2.0BETA/metainfo.xml | 60 +++++++++++++ .../PARSERS/0.2.0BETA/package/.hash | 1 + .../PARSERS/0.2.0BETA/package/archive.zip | Bin 0 -> 4129 bytes .../0.2.0BETA/package/scripts/params.py | 27 ++++++ .../package/scripts/parsers_master.py | 84 ++++++++++++++++++ .../HDP/2.4/services/PARSERS/metainfo.xml | 23 +++++ 7 files changed, 226 insertions(+) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml new file mode 100755 index 0000000000..018dc09600 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml @@ -0,0 +1,31 @@ + + + + + + metron_home + /usr/metron/0.2.0BETA + Metron home directory + + + parsers + bluecoat,bro,snort,squid,websphere,yaf + Metron parsers to deploy + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..ea3479d615 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -0,0 +1,60 @@ + + + + 2.0 + + + PARSERS + Parsers + Parsers + 0.2.0BETA + + + PARSER_MASTER + Parsers Master-Eligible Node + MASTER + 1 + + + PYTHON + 600 + + + + + + any + + + metron-parsers + + + + + + + PYTHON + 300 + + + parsers-env + + true + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash new file mode 100644 index 0000000000..eb3e6503db --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash @@ -0,0 +1 @@ +cae228d81fcb59299a9f086e37cf9c8dbb6aba7d \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..220aa73c79f8832080d0718c5a1a6104c7065b54 GIT binary patch literal 4129 zcmeHKPmkL~6koapA#*@Ty>NSYk&@7jQ;^_L6r{A>RikZ_Xp>Tvwz4Mn#2KwUV`e5z z)2c!}03QO0FTl0pTlEug;k_9<*|b}Vs&b-UC0UQ(|3AO?JUZGve_@BNXMgPWzxnXb zXMgSP?9k^EJTr+Fr8S+BX;zr1Tz%tEw!KF13 zz?gFgLixm-rfB_vWh>*PMTrgH8Feo;##UzgV5%%{Wy2?!B2-(2uAWX8#)wG>dt>O2L%7l#_s9Fe z?f&HY@YV!w_l}QygGv8j48vo%IviZW8{9&odb~I2> z?06alb+OdSsT#-_Q}RU2M1mg4Y{4=PnR>vrq+%$!E`*`%88j^oa#0BDD#n~jcqK6k zT;T^ZtqP2qQ45{$xL^`xQpHBOm};g4H<9lnvQ23VW7kzdum^@~R856H6WKz0O@Y6K zZbcYn+{XT_*$RS!Tdia~R|Q8Be}1>IEuwM%Zo|`U{&O6S`vLYcsx|+Tx1vVA(Hbd! z`~7_X*RT0cpMSKoL!VE#k%FXS7>wl_5#t>;Y#QJlHaK`k{P^EvLxKY@szCz*VukHe z)BthtkS7bv-yB(jBxeYRBj4&x4>+yMekS;qj&mHG2m%9|z;0Hgd z5z^&@*D6Mu$RH|uZB74ENhGPI7Scka?*%*9zv%#}v9^704<^HFLumUEycBs3Q;wsT z<4D!KP?p1LLBL)rXCY@PP1#%}EMH5hGSv);w7sfbfkbnL_=mh<>t*82Q|LH(uOyro zDuwnjsP^)reho2C<_hkGcW)j{j)#Nxr3k0`m4iv|{yiD4ZJ_AT6-rj{?oBsKU`-0R zBb?y)GH0?(nc)kacWGV0rD9|HnbcXC;R2YPbT6Ixm`hqK((ae{Lii3k+$J52Ar*`= zQR-#8I}G63`WNzATf51stH`RJusXamO)9CEgxpY=P+z>&6y1oai(@7vZk4!U;G(7g z=Wxf&X_=)x;)kP(ZPO2XP}f7%2B&BBMF`#A{OBakQFS%ZvGwQBZeyA`jBbsuLtDV( z$M8t06V7oRsmRb5Ty38`f%c>A$CwX1u46>qguSHgrkD0%QD&N@JhrQncd6aqw7q=g zth;MWiq7mUTD7Xk;+FU#N^DSt1XiJ5r`Vl&C*q_?5uyGemd;l~0o*xat4futY()_b z;bLXMmf)&>oaMCdHO?m6IjG&I~LRX9li)jSa*ZJpFHT z;$u*OdKEGsm{+4Pf^eNvGO&ube@kkHc%%zBAz(ZOV(Sp9^32@V9Z(BAh4UCGyD*zD zyuCU^vGp5Jjq}bk>IWRM+a2+i)E;}elzD=KwO&C}l+ZPfq1%Np#$8y*I1YW?kE)s6 zinyQhxYLi0-aB_;_tmfA^m /etc/yum.repos.d/local.repo") + + print 'Install packages' + self.install_packages(env) + + print 'Load parser config' + ## TODO find where to get/share the ZK configs + # Execute(params.metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }}") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + print 'Start the Master' + for parser in self.parser_list: + ## TODO + # start_cmd = params.metron_home + "/bin/start_parser_topology.sh " + parser + print 'Starting ' + parser + # Execute(start_cmd) + print 'Finished starting parser topologies' + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + parser_list = params.parsers.replace(' ', '').split('') + stop_cmd = format("storm kill ") + print 'Stop the Master' + Execute(stop_cmd) + + def status(self, env): + import params + env.set_params(params) + print 'Status of the Master' + + def restart(self, env): + import params + env.set_params(params) + print 'Restarting the Master' + +if __name__ == "__main__": + Parsers().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml new file mode 100644 index 0000000000..c0fc445d81 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml @@ -0,0 +1,23 @@ + + + 2.0 + + + PARSERS + Parsers + Parsers + 0.2.0BETA + common-services/PARSERS/0.2.0BETA + + + any + + + metron-parsers + + + + + + + From 2a6f97830f72151d3813ac0a41d04ef31410fe87 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 9 Aug 2016 18:14:10 -0400 Subject: [PATCH 21/82] add parser and common-services --- metron-deployment/packaging/ambari/.gitignore | 3 + .../{metron.xml => metron-parsers.xml} | 26 +++++ .../PARSERS/0.2.0BETA/metainfo.xml | 33 ++++++- .../PARSERS/0.2.0BETA/package/.hash | 1 - .../PARSERS/0.2.0BETA/package/archive.zip | Bin 4129 -> 0 bytes .../0.2.0BETA/package/scripts/params.py | 15 ++- .../0.2.0BETA/package/scripts/params_linux.py | 89 ++++++++++++++++++ .../package/scripts/params_windows.py | 20 ++++ .../0.2.0BETA/package/scripts/parsers.py | 33 +++++++ .../package/scripts/parsers_master.py | 54 +++++++---- .../package/scripts/service_check.py | 41 ++++++++ .../HDP/2.4/services/PARSERS/metainfo.xml | 12 --- 12 files changed, 290 insertions(+), 37 deletions(-) create mode 100644 metron-deployment/packaging/ambari/.gitignore rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/{metron.xml => metron-parsers.xml} (56%) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip mode change 100755 => 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore new file mode 100644 index 0000000000..db8ded0080 --- /dev/null +++ b/metron-deployment/packaging/ambari/.gitignore @@ -0,0 +1,3 @@ +archive.zip +*.hash + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml similarity index 56% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml index 018dc09600..f067776f87 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml @@ -22,10 +22,36 @@ metron_home /usr/metron/0.2.0BETA Metron home directory + Metron home parsers bluecoat,bro,snort,squid,websphere,yaf Metron parsers to deploy + Metron parsers + + + metron_apps_hdfs_dir + /apps/metron + Metron apps HDFS dir + Metron apps HDFS dir + + + metron_zookeeper_config_dir + config/zookeeper + Metron Zookeeper config dir. Relative path to Metron home. + Metron Zookeeper config dir + + + metron_user + metron + USER + The user for Metron + Metron User + + + metron_group + metron + The group for Metron diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index ea3479d615..d33720699b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -23,12 +23,30 @@ Parsers Parsers 0.2.0BETA + PARSER_MASTER Parsers Master-Eligible Node MASTER 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + PYTHON @@ -36,25 +54,36 @@ + any + metron-common metron-parsers + PYTHON 300 + + + HDFS + KAFKA + STORM + ZOOKEEPER + + - parsers-env + metron-parsers - true + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash deleted file mode 100644 index eb3e6503db..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/.hash +++ /dev/null @@ -1 +0,0 @@ -cae228d81fcb59299a9f086e37cf9c8dbb6aba7d \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/archive.zip deleted file mode 100644 index 220aa73c79f8832080d0718c5a1a6104c7065b54..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4129 zcmeHKPmkL~6koapA#*@Ty>NSYk&@7jQ;^_L6r{A>RikZ_Xp>Tvwz4Mn#2KwUV`e5z z)2c!}03QO0FTl0pTlEug;k_9<*|b}Vs&b-UC0UQ(|3AO?JUZGve_@BNXMgPWzxnXb zXMgSP?9k^EJTr+Fr8S+BX;zr1Tz%tEw!KF13 zz?gFgLixm-rfB_vWh>*PMTrgH8Feo;##UzgV5%%{Wy2?!B2-(2uAWX8#)wG>dt>O2L%7l#_s9Fe z?f&HY@YV!w_l}QygGv8j48vo%IviZW8{9&odb~I2> z?06alb+OdSsT#-_Q}RU2M1mg4Y{4=PnR>vrq+%$!E`*`%88j^oa#0BDD#n~jcqK6k zT;T^ZtqP2qQ45{$xL^`xQpHBOm};g4H<9lnvQ23VW7kzdum^@~R856H6WKz0O@Y6K zZbcYn+{XT_*$RS!Tdia~R|Q8Be}1>IEuwM%Zo|`U{&O6S`vLYcsx|+Tx1vVA(Hbd! z`~7_X*RT0cpMSKoL!VE#k%FXS7>wl_5#t>;Y#QJlHaK`k{P^EvLxKY@szCz*VukHe z)BthtkS7bv-yB(jBxeYRBj4&x4>+yMekS;qj&mHG2m%9|z;0Hgd z5z^&@*D6Mu$RH|uZB74ENhGPI7Scka?*%*9zv%#}v9^704<^HFLumUEycBs3Q;wsT z<4D!KP?p1LLBL)rXCY@PP1#%}EMH5hGSv);w7sfbfkbnL_=mh<>t*82Q|LH(uOyro zDuwnjsP^)reho2C<_hkGcW)j{j)#Nxr3k0`m4iv|{yiD4ZJ_AT6-rj{?oBsKU`-0R zBb?y)GH0?(nc)kacWGV0rD9|HnbcXC;R2YPbT6Ixm`hqK((ae{Lii3k+$J52Ar*`= zQR-#8I}G63`WNzATf51stH`RJusXamO)9CEgxpY=P+z>&6y1oai(@7vZk4!U;G(7g z=Wxf&X_=)x;)kP(ZPO2XP}f7%2B&BBMF`#A{OBakQFS%ZvGwQBZeyA`jBbsuLtDV( z$M8t06V7oRsmRb5Ty38`f%c>A$CwX1u46>qguSHgrkD0%QD&N@JhrQncd6aqw7q=g zth;MWiq7mUTD7Xk;+FU#N^DSt1XiJ5r`Vl&C*q_?5uyGemd;l~0o*xat4futY()_b z;bLXMmf)&>oaMCdHO?m6IjG&I~LRX9li)jSa*ZJpFHT z;$u*OdKEGsm{+4Pf^eNvGO&ube@kkHc%%zBAz(ZOV(Sp9^32@V9Z(BAh4UCGyD*zD zyuCU^vGp5Jjq}bk>IWRM+a2+i)E;}elzD=KwO&C}l+ZPfq1%Np#$8y*I1YW?kE)s6 zinyQhxYLi0-aB_;_tmfA^m /etc/yum.repos.d/local.repo") - print 'Install packages' + print 'Install RPM packages' self.install_packages(env) + print 'Upload grok patterns' + parsers_init() + + print 'Setup Kafka topics' + for parser in parser_list: + Execute(format("""/usr/hdp/current/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {zookeeper_url} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""").format(parser,1,1,10 * 1024 * 1024 * 1024)) +# {{ kafka_home }}/bin/kafka-topics.sh \ +# --zookeeper {{ zookeeper_url }} \ +# --create \ +# --topic {{ item.topic }} \ +# --partitions {{ item.num_partitions }} \ +# --replication-factor {{ item.replication_factor }} \ +# --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }} + print 'Load parser config' - ## TODO find where to get/share the ZK configs - # Execute(params.metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }}") + #Execute(format("{metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {zookeeper_url}") + Execute(format("{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_url}")) def start(self, env, upgrade_type=None): import params env.set_params(params) + parser_list = params.parsers.replace(' ', '').split(',') print 'Start the Master' - for parser in self.parser_list: - ## TODO - # start_cmd = params.metron_home + "/bin/start_parser_topology.sh " + parser + for parser in parser_list: print 'Starting ' + parser - # Execute(start_cmd) + start_cmd = params.metron_home + '/bin/start_parser_topology.sh -s ' + parser + ' -z localhost:2181' + Execute(start_cmd) print 'Finished starting parser topologies' def stop(self, env, upgrade_type=None): import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split('') - stop_cmd = format("storm kill ") - print 'Stop the Master' - Execute(stop_cmd) + parser_list = params.parsers.replace(' ', '').split(',') + print 'Stopping parsers' + for parser in parser_list: + print 'Stopping ' + parser + stop_cmd = 'storm kill ' + parser + Execute(stop_cmd) + print 'Done' def status(self, env): import params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py new file mode 100755 index 0000000000..a363d65746 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from __future__ import print_function + +import sys + +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute +import subprocess + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + doc = '{"name": "Ambari Smoke test"}' + index = "ambari_smoke_test" + + print("Running parsers service check", file=sys.stdout) + exit(0) + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml old mode 100644 new mode 100755 index c0fc445d81..6280214962 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml @@ -4,20 +4,8 @@ PARSERS - Parsers - Parsers 0.2.0BETA common-services/PARSERS/0.2.0BETA - - - any - - - metron-parsers - - - - From c2735510f1949b3682a0ecb8473d791c56e71315 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 10 Aug 2016 18:12:04 -0400 Subject: [PATCH 22/82] finish commands. refactor --- .../0.2.0BETA/package/scripts/commands.py | 126 ++++++++++++++++++ .../package/scripts/{ => params}/params.py | 0 .../scripts/{ => params}/params_linux.py | 22 +-- .../scripts/{ => params}/params_windows.py | 0 .../0.2.0BETA/package/scripts/parsers.py | 33 ----- .../package/scripts/parsers_master.py | 87 ++++-------- .../package/scripts/service_check.py | 2 - 7 files changed, 165 insertions(+), 105 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params.py (100%) mode change 100644 => 100755 rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params_linux.py (83%) rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{ => params}/params_windows.py (100%) mode change 100644 => 100755 delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py new file mode 100755 index 0000000000..ed2f880b3c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format + + +# Wrap major operations and functionality in this class +class Commands: + __params = None + __parser_list = None + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__parser_list = self.__get_parsers(params) + + # get list of parsers + def __get_parsers(self, params): + return params.parsers.replace(' ', '').split(',') + + def init_parsers(self): + Logger.info( + "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, + self.__params.metron_apps_dir)) + self.__params.HdfsResource(self.__params.metron_apps_dir, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + mode=0775, + source=self.__params.local_grok_patterns_dir) + Logger.info("Done initializing parser configuration") + + def get_parser_list(self): + return self.__parser_list + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + # TODO get HDP home from params + command_template = format("""{hadoop_home_dir}/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {zookeeper_quorum} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""") + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = 10 + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating main topics for parsers") + for parser_name in self.get_parser_list(): + Logger.info("Creating topic'{}'".format(parser_name)) + Execute(command_template.format(parser_name, num_partitions, replication_factor, retention_bytes)) + Logger.info("Creating topics for error handling") + Execute(command_template.format("parser_invalid", num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format("parser_error", num_partitions, replication_factor, retention_bytes)) + Logger.info("Done creating Kafka topics") + + def init_parser_config(self): + Logger.info('Loading parser config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) + + def start_parser_topologies(self): + Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) + start_cmd_template = format("""{metron_home}/bin/start_parser_topology.sh \ + -s {} \ + -z {zookeeper_quorum}""") + for parser in self.get_parser_list(): + Logger.info('Starting ' + parser) + Execute(start_cmd_template.format(parser)) + + Logger.info('Finished starting parser topologies') + + def stop_parser_topologies(self): + Logger.info('Stopping parsers') + for parser in self.get_parser_list(): + Logger.info('Stopping ' + parser) + stop_cmd = 'storm kill ' + parser + Execute(stop_cmd) + Logger.info('Done stopping parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py old mode 100644 new mode 100755 similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py similarity index 83% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index 2a3da98150..bc982cb06b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -35,8 +35,11 @@ hostname = config['hostname'] metron_home = config['configurations']['metron-parsers']['metron_home'] parsers = config['configurations']['metron-parsers']['parsers'] +metron_user = config['configurations']['metron-parsers']['metron_user'] +metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +yum_repo_type = 'local' # hadoop params hadoop_home_dir = stack_select.get_hadoop_dir("home") @@ -44,14 +47,17 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir() # zookeeper -# zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] -zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) -zookeeper_url = format("{hostname}:{zookeeper_port}") - -# kafka -kafka_host = config['clusterHostInfo']['kafka_broker_hosts'][0] -# TODO -kafka_port = 6667 +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py old mode 100644 new mode 100755 similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params_windows.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py deleted file mode 100644 index 66aa6ed3dc..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - - -def parsers_init(): - import params - params.HdfsResource(params.metron_apps_dir, - type="directory", - action="create_on_execute", - owner=params.metron_user, - mode=0775, - source=params.local_grok_patterns_dir) - - -def get_parsers(params): - return params.parsers.replace(' ', '').split(',') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index dd708b6b8a..ecdd4acd03 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -17,88 +17,51 @@ """ +from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions import format from resource_management.libraries.script import Script -from parsers import parsers_init +from commands import Commands -class Parsers(Script): + +class ParsersMaster(Script): def install(self, env): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - - print 'Install the Master' - ## TODO - this will become a remote yum repo instead of local - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - Execute("echo \"[METRON-0.2.0BETA]\n" - "name=Metron 0.2.0BETA packages\n" - "baseurl=file:///localrepo\n" - "gpgcheck=0\n" - "enabled=1\" > /etc/yum.repos.d/local.repo") - - print 'Install RPM packages' + commands = Commands(params) + commands.setup_repo() + Logger.info('Install RPM packages') self.install_packages(env) - - print 'Upload grok patterns' - parsers_init() - - print 'Setup Kafka topics' - for parser in parser_list: - Execute(format("""/usr/hdp/current/kafka-broker/bin/kafka-topics.sh \ - --zookeeper {zookeeper_url} \ - --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""").format(parser,1,1,10 * 1024 * 1024 * 1024)) -# {{ kafka_home }}/bin/kafka-topics.sh \ -# --zookeeper {{ zookeeper_url }} \ -# --create \ -# --topic {{ item.topic }} \ -# --partitions {{ item.num_partitions }} \ -# --replication-factor {{ item.replication_factor }} \ -# --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }} - - print 'Load parser config' - #Execute(format("{metron_home + "/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {zookeeper_url}") - Execute(format("{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_url}")) + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() def start(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - print 'Start the Master' - for parser in parser_list: - print 'Starting ' + parser - start_cmd = params.metron_home + '/bin/start_parser_topology.sh -s ' + parser + ' -z localhost:2181' - Execute(start_cmd) - print 'Finished starting parser topologies' + commands = Commands(params) + commands.start_parser_topologies() def stop(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - parser_list = params.parsers.replace(' ', '').split(',') - print 'Stopping parsers' - for parser in parser_list: - print 'Stopping ' + parser - stop_cmd = 'storm kill ' + parser - Execute(stop_cmd) - print 'Done' + commands = Commands(params) + commands.stop_parser_topologies() def status(self, env): - import params + from params import params env.set_params(params) - print 'Status of the Master' + Logger.info('Status of the Master') def restart(self, env): - import params + from params import params env.set_params(params) - print 'Restarting the Master' + Logger.info('Restarting the parser topologies') + self.stop(env) + self.start(env) + Logger.info('Done restarting the parser topologies') if __name__ == "__main__": - Parsers().execute() + ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index a363d65746..9746ea4c7b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -22,8 +22,6 @@ import sys from resource_management.libraries.script import Script -from resource_management.core.resources.system import Execute -import subprocess class ServiceCheck(Script): From 959991b6a0497acd93422da4c5cc8b09a43cd48c Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 12 Aug 2016 12:00:13 -0400 Subject: [PATCH 23/82] fixes --- .../PARSERS/0.2.0BETA/metainfo.xml | 2 ++ .../0.2.0BETA/package/scripts/commands.py | 6 ++++++ .../package/scripts/params/__init__.py | 0 .../package/scripts/parsers_master.py | 20 +++++++++---------- .../services/PARSERS/metainfo.xml | 0 .../services/PARSERS/role_command_order.json | 8 ++++++++ 6 files changed, 26 insertions(+), 10 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py rename metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/{2.4 => 2.3}/services/PARSERS/metainfo.xml (100%) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index d33720699b..2291f622bb 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -61,6 +61,8 @@ metron-common + + metron-parsers diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index ed2f880b3c..118ddae4c4 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -124,3 +124,9 @@ def stop_parser_topologies(self): stop_cmd = 'storm kill ' + parser Execute(stop_cmd) Logger.info('Done stopping parser topologies') + + def restart_parser_topologies(self): + Logger.info('Restarting the parser topologies') + self.stop_parser_topologies() + self.start_parser_topologies() + Logger.info('Done restarting the parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index ecdd4acd03..4d0107fb45 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -18,15 +18,12 @@ """ from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import format from resource_management.libraries.script import Script from commands import Commands class ParsersMaster(Script): - def install(self, env): from params import params env.set_params(params) @@ -34,14 +31,18 @@ def install(self, env): commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) - commands.init_parsers() - commands.init_kafka_topics() - commands.init_parser_config() + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) def start(self, env, upgrade_type=None): from params import params env.set_params(params) commands = Commands(params) + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): @@ -58,10 +59,9 @@ def status(self, env): def restart(self, env): from params import params env.set_params(params) - Logger.info('Restarting the parser topologies') - self.stop(env) - self.start(env) - Logger.info('Done restarting the parser topologies') + commands = Commands(params) + commands.restart_parser_topologies() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/PARSERS/metainfo.xml rename to metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json new file mode 100755 index 0000000000..dcee302c46 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -0,0 +1,8 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "PARSER_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] + } +} From fe7f46822104d6b313ce80516916dcde9a0add4e Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 12 Aug 2016 12:18:18 -0400 Subject: [PATCH 24/82] config fixes --- .../0.2.0BETA/package/scripts/commands.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 118ddae4c4..f1e892d5d6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -80,14 +80,13 @@ def remote_repo(): def init_kafka_topics(self): Logger.info('Creating Kafka topics') - # TODO get HDP home from params - command_template = format("""{hadoop_home_dir}/kafka-broker/bin/kafka-topics.sh \ - --zookeeper {zookeeper_quorum} \ + command_template = """{}/kafka-broker/bin/kafka-topics.sh \ + --zookeeper {} \ --create \ --topic {} \ --partitions {} \ --replication-factor {} \ - --config retention.bytes={}""") + --config retention.bytes={}""" num_partitions = 1 replication_factor = 1 retention_gigabytes = 10 @@ -95,10 +94,24 @@ def init_kafka_topics(self): Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): Logger.info("Creating topic'{}'".format(parser_name)) - Execute(command_template.format(parser_name, num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + parser_name, + num_partitions, + replication_factor, + retention_bytes)) Logger.info("Creating topics for error handling") - Execute(command_template.format("parser_invalid", num_partitions, replication_factor, retention_bytes)) - Execute(command_template.format("parser_error", num_partitions, replication_factor, retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + "parser_invalid", + num_partitions, + replication_factor, + retention_bytes)) + Execute(command_template.format(self.__params.hadoop_home_dir, + self.__params.zookeeper_quorum, + "parser_error", + num_partitions, replication_factor, + retention_bytes)) Logger.info("Done creating Kafka topics") def init_parser_config(self): @@ -108,12 +121,12 @@ def init_parser_config(self): def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) - start_cmd_template = format("""{metron_home}/bin/start_parser_topology.sh \ + start_cmd_template = """{}/bin/start_parser_topology.sh \ -s {} \ - -z {zookeeper_quorum}""") + -z {}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) - Execute(start_cmd_template.format(parser)) + Execute(start_cmd_template.format(self.__params.metron_home, parser, self.__params.zookeeper_quorum)) Logger.info('Finished starting parser topologies') From 8bab89166952c736962fe07a7478829575a5fb61 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 16 Aug 2016 10:10:27 -0400 Subject: [PATCH 25/82] incremental progress. quicklinks. kafka home --- .../PARSERS/0.2.0BETA/metainfo.xml | 18 ++++++++++-- .../0.2.0BETA/package/scripts/commands.py | 8 +++--- .../package/scripts/params/params_linux.py | 4 +++ .../0.2.0BETA/quicklinks/quicklinks.json | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 2291f622bb..77adbf8f89 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -20,14 +20,14 @@ PARSERS - Parsers + Metron Parsers Parsers 0.2.0BETA PARSER_MASTER - Parsers Master-Eligible Node + Metron Parser Topologies MASTER 1 true @@ -46,6 +46,13 @@ true + + KAFKA/KAFKA_BROKER + cluster + + true + + @@ -86,6 +93,13 @@ metron-parsers + + + quicklinks.json + true + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index f1e892d5d6..7aefef3065 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -80,7 +80,7 @@ def remote_repo(): def init_kafka_topics(self): Logger.info('Creating Kafka topics') - command_template = """{}/kafka-broker/bin/kafka-topics.sh \ + command_template = """{}/kafka-topics.sh \ --zookeeper {} \ --create \ --topic {} \ @@ -94,20 +94,20 @@ def init_kafka_topics(self): Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): Logger.info("Creating topic'{}'".format(parser_name)) - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, parser_name, num_partitions, replication_factor, retention_bytes)) Logger.info("Creating topics for error handling") - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, "parser_invalid", num_partitions, replication_factor, retention_bytes)) - Execute(command_template.format(self.__params.hadoop_home_dir, + Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, "parser_error", num_partitions, replication_factor, diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index bc982cb06b..be91c497ed 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -19,6 +19,7 @@ """ import functools +import os from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import format @@ -42,9 +43,12 @@ yum_repo_type = 'local' # hadoop params +stack_root = Script.get_stack_root() hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin", "kafka") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json new file mode 100755 index 0000000000..ee1b225084 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} From 43b6d04a8712c7332dd45116113b29ef61ebf8f9 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 16 Aug 2016 17:01:46 -0400 Subject: [PATCH 26/82] incremental fixes --- .../configuration/metron-parsers.xml | 2 +- .../PARSERS/0.2.0BETA/metainfo.xml | 21 ++++++ .../package/scripts/custom_commands.py | 40 +++++++++++ .../package/scripts/params/params_linux.py | 2 +- .../package/scripts/parsers_master.py | 12 ++++ .../PARSERS/0.2.0BETA/service_advisor.py | 69 +++++++++++++++++++ 6 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml index f067776f87..71966eab9f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml @@ -26,7 +26,7 @@ parsers - bluecoat,bro,snort,squid,websphere,yaf + bro,snort,yaf Metron parsers to deploy Metron parsers diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 77adbf8f89..f92d424c87 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -59,6 +59,24 @@ PYTHON 600 + + + KAFKABUILD + + + PYTHON + 600 + + + + ZOOKEEPERBUILD + + + PYTHON + 600 + + + @@ -66,6 +84,9 @@ any + + java-1.8.0-openjdk-devel + metron-common diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py new file mode 100644 index 0000000000..46bb7d3d34 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +from commands import Commands + + +class CustomCommands(Script): + def kafkabuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + def zookeeperbuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_parser_config() + + +if __name__ == "__main__": + CustomCommands().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index be91c497ed..f8b8390245 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -48,7 +48,7 @@ hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin", "kafka") +kafka_bin_dir = os.path.join(kafka_home, "bin") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index 4d0107fb45..8f893c016f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -62,6 +62,18 @@ def restart(self, env): commands = Commands(params) commands.restart_parser_topologies() + def kafkabuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + def zookeeperbuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.init_parser_config() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py new file mode 100644 index 0000000000..2a545da333 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -0,0 +1,69 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) + + +class MetronParsersServiceAdvisor(service_advisor.ServiceAdvisor): + # colocate Metron Parser Master with KAFKA_BROKERs + def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + parsersMasterComponent = [component for component in serviceComponents if + component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"] + parsersMasterComponent = parsersMasterComponent[0] + if not stackAdvisor.isComponentHostsPopulated(parsersMasterComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: + hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"}) + if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) + + def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + masterHosts = self.getHosts(componentsList, "PARSER_MASTER") + expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) + + items = [] + + mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Metron Parsers Master must be installed on Kafka Brokers. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( + len(mismatchHosts), hostsString) + items.append( + {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'}) + + return items From 800322dc948cffbaf03788efc8cf756d8b8cb8cf Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 17 Aug 2016 17:09:44 -0400 Subject: [PATCH 27/82] check service configured on start. fix zk load. fix topology start --- .../PARSERS/0.2.0BETA/metainfo.xml | 3 -- .../0.2.0BETA/package/scripts/commands.py | 34 +++++++++++++--- .../package/scripts/custom_commands.py | 40 ------------------- .../package/scripts/params/params_linux.py | 15 +++++++ .../package/scripts/parsers_master.py | 14 +++++-- .../package/templates/metron-global.json | 1 + .../PARSERS/0.2.0BETA/service_advisor.py | 0 7 files changed, 56 insertions(+), 51 deletions(-) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index f92d424c87..70d3378c41 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -84,9 +84,6 @@ any - - java-1.8.0-openjdk-devel - metron-common diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 7aefef3065..91e3d1d9c9 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -18,8 +18,11 @@ """ +import os + from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import Template from resource_management.libraries.functions import format @@ -27,17 +30,28 @@ class Commands: __params = None __parser_list = None + __configured = False def __init__(self, params): if params is None: raise ValueError("params argument is required for initialization") self.__params = params self.__parser_list = self.__get_parsers(params) + self.__configured = os.path.isfile(self.__params.configured_flag_file) # get list of parsers def __get_parsers(self, params): return params.parsers.replace(' ', '').split(',') + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, @@ -48,6 +62,12 @@ def init_parsers(self): owner=self.__params.metron_user, mode=0775, source=self.__params.local_grok_patterns_dir) + + Logger.info("Creating global.json file") + File(self.__params.metron_zookeeper_config_path + '/global.json', + content=Template("metron-global.json"), + owner=self.__params.metron_user, + mode=0775) Logger.info("Done initializing parser configuration") def get_parser_list(self): @@ -117,16 +137,20 @@ def init_kafka_topics(self): def init_parser_config(self): Logger.info('Loading parser config into ZooKeeper') Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}")) + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) start_cmd_template = """{}/bin/start_parser_topology.sh \ - -s {} \ - -z {}""" + -k {} \ + -z {} \ + -s {}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) - Execute(start_cmd_template.format(self.__params.metron_home, parser, self.__params.zookeeper_quorum)) + Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, + self.__params.zookeeper_quorum, parser)) Logger.info('Finished starting parser topologies') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py deleted file mode 100644 index 46bb7d3d34..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/custom_commands.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -from commands import Commands - - -class CustomCommands(Script): - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - - def zookeeperbuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_parser_config() - - -if __name__ == "__main__": - CustomCommands().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index f8b8390245..539dbf226c 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -40,6 +40,8 @@ metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +#metron_global_config_content = config['configurations']['global.json']['content'] yum_repo_type = 'local' # hadoop params @@ -63,11 +65,24 @@ # last port config zookeeper_quorum += ':' + zookeeper_clientPort +# Kafka +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] local_grok_patterns_dir = format("{metron_home}/patterns") hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") + # for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] hostname = config["hostname"] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py index 8f893c016f..c28ee99745 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py @@ -40,9 +40,11 @@ def start(self, env, upgrade_type=None): from params import params env.set_params(params) commands = Commands(params) - commands.init_parsers() - commands.init_kafka_topics() - commands.init_parser_config() + if not commands.is_configured(): + commands.init_parsers() + commands.init_kafka_topics() + commands.init_parser_config() + commands.set_configured() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): @@ -74,6 +76,12 @@ def zookeeperbuild(self, env, upgrade_type=None): commands = Commands(params) commands.init_parser_config() + def starttopologies(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = Commands(params) + commands.start_parser_topologies() + if __name__ == "__main__": ParsersMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json @@ -0,0 +1 @@ +{} diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py old mode 100644 new mode 100755 From 0a723a458154e87f94210756cc3e2e49d3a805ad Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 18 Aug 2016 16:21:33 -0400 Subject: [PATCH 28/82] service check --- .../0.2.0BETA/package/scripts/commands.py | 53 +++++++++++++++++++ .../package/scripts/params/params_linux.py | 1 - .../package/scripts/service_check.py | 11 ++-- .../PARSERS/0.2.0BETA/service_advisor.py | 11 ++-- .../services/PARSERS/role_command_order.json | 20 ++++--- 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 91e3d1d9c9..60bd3766ee 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -19,6 +19,9 @@ """ import os +import re +import subprocess +import time from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File @@ -52,6 +55,14 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) + # Possible storm topology status states + # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html + class StormStatus: + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + KILLED = "KILLED" + REBALANCING = "REBALANCING" + def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, @@ -165,5 +176,47 @@ def stop_parser_topologies(self): def restart_parser_topologies(self): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() + attempt_count = 0 + while self.topologies_active(): + if attempt_count > 2: + raise Exception("Unable to stop topologies") + attempt_count += 1 + time.sleep(10) self.start_parser_topologies() Logger.info('Done restarting the parser topologies') + + def topologies_active(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + all_active = False + if stdout_lines: + all_active = True + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + parser_found=False + is_active=False + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + status = items[1] + parser_found=True + is_active = self.__is_active(status) + all_active &= parser_found and is_active + return all_active + + def __get_status_lines(self, lines): + status_lines=[] + do_stat = False + skipped = 0 + for line in lines: + if line.startswith("Topology_name"): + do_stat = True + if do_stat and skipped == 2: + status_lines += [line] + elif do_stat: + skipped += 1 + return status_lines + + def __is_active(self, status): + return status == self.StormStatus.ACTIVE diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index 539dbf226c..b995e722b5 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -41,7 +41,6 @@ metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -#metron_global_config_content = config['configurations']['global.json']['content'] yum_repo_type = 'local' # hadoop params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 9746ea4c7b..6fcce1680a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -19,21 +19,20 @@ """ from __future__ import print_function -import sys - from resource_management.libraries.script import Script +from commands import Commands + class ServiceCheck(Script): def service_check(self, env): import params env.set_params(params) + commands = Commands(params) + all_found = commands.topologies_active() - doc = '{"name": "Ambari Smoke test"}' - index = "ambari_smoke_test" + exit(int(all_found)) - print("Running parsers service check", file=sys.stdout) - exit(0) if __name__ == "__main__": ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py index 2a545da333..ddf58d5d9a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -32,13 +32,12 @@ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) -class MetronParsersServiceAdvisor(service_advisor.ServiceAdvisor): +class PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): # colocate Metron Parser Master with KAFKA_BROKERs - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def TODO_colocateService(self, hostsComponentsMap, serviceComponents): parsersMasterComponent = [component for component in serviceComponents if - component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"] - parsersMasterComponent = parsersMasterComponent[0] - if not stackAdvisor.isComponentHostsPopulated(parsersMasterComponent): + component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0] + if not self.isComponentHostsPopulated(parsersMasterComponent): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: @@ -46,7 +45,7 @@ def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def TODO_getServiceComponentLayoutValidations(self, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json index dcee302c46..1d35b16033 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -1,8 +1,16 @@ { - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "PARSER_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START"] - } + "_comment": "Record format:", + "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps": { + "_comment": "dependencies for Metron Parsers", + "PARSER_MASTER-START": [ + "NAMENODE-START", + "ZOOKEEPER_SERVER-START", + "KAFKA_BROKER-START", + "STORM_REST_API-START" + ], + "PARSER_MASTER_SERVICE_CHECK-SERVICE_CHECK": [ + "PARSER_MASTER-START" + ] + } } From 2fef17dc31194be8297f1e930e70738064b62ac3 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 18 Aug 2016 17:40:12 -0400 Subject: [PATCH 29/82] trying to fix service check --- .../PARSERS/0.2.0BETA/package/scripts/params/params_linux.py | 3 +-- .../PARSERS/0.2.0BETA/package/scripts/service_check.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index b995e722b5..d9fdc6f857 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -39,7 +39,7 @@ metron_user = config['configurations']['metron-parsers']['metron_user'] metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format("{metron_home}/{metron_zookeeper_config_dir}") +metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' yum_repo_type = 'local' @@ -81,7 +81,6 @@ local_grok_patterns_dir = format("{metron_home}/patterns") hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") - # for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] hostname = config["hostname"] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 6fcce1680a..484b62806b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,7 +26,7 @@ class ServiceCheck(Script): def service_check(self, env): - import params + from params import params env.set_params(params) commands = Commands(params) all_found = commands.topologies_active() From 4d228df31fe04ca09f356fe495370419221daef6 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 11:25:07 -0400 Subject: [PATCH 30/82] cleanup/refactor. fix service check ordering --- .../PARSERS/0.2.0BETA/metainfo.xml | 23 +--- .../0.2.0BETA/package/scripts/commands.py | 41 ++++--- .../package/scripts/params/params_linux.py | 4 +- .../package/scripts/params/status_params.py | 113 ++++++++++++++++++ .../{parsers_master.py => parser_master.py} | 30 ++++- .../package/scripts/service_check.py | 16 ++- .../package/templates/metron-global.json | 0 .../PARSERS/0.2.0BETA/service_advisor.py | 2 +- .../services/PARSERS/role_command_order.json | 3 +- 9 files changed, 185 insertions(+), 47 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py rename metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/{parsers_master.py => parser_master.py} (74%) mode change 100644 => 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml index 70d3378c41..f1e5ab7d90 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml @@ -55,28 +55,13 @@ - + PYTHON 600 - - - KAFKABUILD - - - PYTHON - 600 - - - - ZOOKEEPERBUILD - - - PYTHON - 600 - - - + + metron-parsers + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py index 60bd3766ee..6b33e55a55 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py @@ -177,36 +177,49 @@ def restart_parser_topologies(self): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() attempt_count = 0 - while self.topologies_active(): + while self.topologies_exist(): if attempt_count > 2: - raise Exception("Unable to stop topologies") + raise Exception("Unable to kill topologies") attempt_count += 1 time.sleep(10) self.start_parser_topologies() Logger.info('Done restarting the parser topologies') - def topologies_active(self): + def topologies_exist(self): cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = cmd_open.communicate() stdout_lines = stdout.splitlines() - all_active = False if stdout_lines: - all_active = True status_lines = self.__get_status_lines(stdout_lines) for parser in self.get_parser_list(): - parser_found=False - is_active=False + for line in status_lines: + items = re.sub('[\s]+', ' ', line).split() + if items and items[0] == parser: + return True + return False + + def topologies_running(self): + cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = cmd_open.communicate() + stdout_lines = stdout.splitlines() + all_running = False + if stdout_lines: + all_running = True + status_lines = self.__get_status_lines(stdout_lines) + for parser in self.get_parser_list(): + parser_found = False + is_running = False for line in status_lines: items = re.sub('[\s]+', ' ', line).split() if items and items[0] == parser: status = items[1] - parser_found=True - is_active = self.__is_active(status) - all_active &= parser_found and is_active - return all_active + parser_found = True + is_running = self.__is_running(status) + all_running &= parser_found and is_running + return all_running def __get_status_lines(self, lines): - status_lines=[] + status_lines = [] do_stat = False skipped = 0 for line in lines: @@ -218,5 +231,5 @@ def __get_status_lines(self, lines): skipped += 1 return status_lines - def __is_active(self, status): - return status == self.StormStatus.ACTIVE + def __is_running(self, status): + return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py index d9fdc6f857..de7dfc9ff2 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py @@ -34,12 +34,14 @@ config = Script.get_config() hostname = config['hostname'] +#print(config['configurations']) metron_home = config['configurations']['metron-parsers']['metron_home'] parsers = config['configurations']['metron-parsers']['parsers'] metron_user = config['configurations']['metron-parsers']['metron_user'] metron_group = config['configurations']['metron-parsers']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' yum_repo_type = 'local' diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py new file mode 100755 index 0000000000..9653d07feb --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import functools +import os + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +hostname = config['hostname'] + +metron_home = config['configurations']['metron-parsers']['metron_home'] +parsers = config['configurations']['metron-parsers']['parsers'] +metron_user = config['configurations']['metron-parsers']['metron_user'] +metron_group = config['configurations']['metron-parsers']['metron_group'] +metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') +configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +yum_repo_type = 'local' + +# hadoop params +stack_root = Script.get_stack_root() +hadoop_home_dir = stack_select.get_hadoop_dir("home") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") + +# zookeeper +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort + +# Kafka +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + +metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] + +local_grok_patterns_dir = format("{metron_home}/patterns") +hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") + +# for create_hdfs_directory +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") + +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + hadoop_bin_dir=hadoop_bin_dir, + hadoop_conf_dir=hadoop_conf_dir, + principal_name=hdfs_principal_name, + hdfs_site=hdfs_site, + default_fs=default_fs, + immutable_paths=get_not_managed_resources(), + dfs_type=dfs_type +) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py similarity index 74% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py index c28ee99745..1290627567 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parsers_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py @@ -17,13 +17,19 @@ """ +from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script from commands import Commands -class ParsersMaster(Script): +class ParserMaster(Script): + def get_component_name(self): + # TODO add this at some point - currently will cause problems with hdp-select + #return "parser-master" + pass + def install(self, env): from params import params env.set_params(params) @@ -54,9 +60,23 @@ def stop(self, env, upgrade_type=None): commands.stop_parser_topologies() def status(self, env): - from params import params - env.set_params(params) - Logger.info('Status of the Master') + from params import status_params + env.set_params(status_params) + commands = Commands(status_params) + if not commands.topologies_running(): + raise ComponentIsNotRunning() + +# from params import status_params +# env.set_params(params) +# commands = Commands(params) +# if not commands.topologies_running(): +# raise ComponentIsNotRunning() + +# from params import status_params +# env.set_params(status_params) +# commands = Commands(params) +# if not commands.topologies_running(): +# raise ComponentIsNotRunning() def restart(self, env): from params import params @@ -84,4 +104,4 @@ def starttopologies(self, env, upgrade_type=None): if __name__ == "__main__": - ParsersMaster().execute() + ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index 484b62806b..f77a09f8f6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,12 +26,16 @@ class ServiceCheck(Script): def service_check(self, env): - from params import params - env.set_params(params) - commands = Commands(params) - all_found = commands.topologies_active() - - exit(int(all_found)) + print("service_check CALLED") +# from params import params +# env.set_params(params) +# from params import params +# env.set_params(params) +# commands = Commands(params) +# all_found = commands.topologies_active() +# +# exit(int(all_found)) + exit(0) if __name__ == "__main__": diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json old mode 100644 new mode 100755 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py index ddf58d5d9a..84b466a353 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py @@ -32,7 +32,7 @@ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) -class PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): +class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): # colocate Metron Parser Master with KAFKA_BROKERs def TODO_colocateService(self, hostsComponentsMap, serviceComponents): parsersMasterComponent = [component for component in serviceComponents if diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json index 1d35b16033..c8616469b6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json @@ -9,7 +9,8 @@ "KAFKA_BROKER-START", "STORM_REST_API-START" ], - "PARSER_MASTER_SERVICE_CHECK-SERVICE_CHECK": [ + "_comment": "_SERVICE_CHECK-SERVICE_CHECK", + "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [ "PARSER_MASTER-START" ] } From 23460732e5dfeb2206f629b777ca8225d6aad456 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 12:07:02 -0400 Subject: [PATCH 31/82] fix service check --- .../0.2.0BETA/package/scripts/service_check.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py index f77a09f8f6..6ef7e4aa14 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py @@ -26,16 +26,14 @@ class ServiceCheck(Script): def service_check(self, env): - print("service_check CALLED") -# from params import params -# env.set_params(params) -# from params import params -# env.set_params(params) -# commands = Commands(params) -# all_found = commands.topologies_active() -# -# exit(int(all_found)) - exit(0) + from params import params + env.set_params(params) + commands = Commands(params) + all_found = commands.topologies_running() + if all_found: + exit(0) + else: + exit(1) if __name__ == "__main__": From 9ee5c8a70c4aba832d74b83c0ca74db12f319eb6 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 19 Aug 2016 17:34:14 -0400 Subject: [PATCH 32/82] METRON-383 Ambari install for Metron Parser topologies. --- .../package/scripts/params/__init__.py | 18 ++++ .../package/scripts/params/status_params.py | 95 +----------------- .../package/scripts/parser_master.py | 32 +----- .../PARSERS/0.2.0BETA/service_advisor.pyc | Bin 0 -> 3700 bytes .../HDP/2.3/services/PARSERS/metainfo.xml | 16 +++ 5 files changed, 37 insertions(+), 124 deletions(-) create mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py index e69de29bb2..242460e266 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py index 9653d07feb..ee83414f0c 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py @@ -18,96 +18,5 @@ """ -import functools -import os - -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import format -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.script import Script - -# server configurations -config = Script.get_config() - -hostname = config['hostname'] - -metron_home = config['configurations']['metron-parsers']['metron_home'] -parsers = config['configurations']['metron-parsers']['parsers'] -metron_user = config['configurations']['metron-parsers']['metron_user'] -metron_group = config['configurations']['metron-parsers']['metron_group'] -metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -yum_repo_type = 'local' - -# hadoop params -stack_root = Script.get_stack_root() -hadoop_home_dir = stack_select.get_hadoop_dir("home") -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hadoop_conf_dir = conf_select.get_hadoop_conf_dir() -kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin") - -# zookeeper -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -has_zk_host = not len(zk_hosts) == 0 -zookeeper_quorum = None -if has_zk_host: - if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: - zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] - else: - zookeeper_clientPort = '2181' - zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) - # last port config - zookeeper_quorum += ':' + zookeeper_clientPort - -# Kafka -kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) -has_kafka_host = not len(kafka_hosts) == 0 -kafka_brokers = None -if has_kafka_host: - if 'port' in config['configurations']['kafka-broker']: - kafka_broker_port = config['configurations']['kafka-broker']['port'] - else: - kafka_broker_port = '6667' - kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) - kafka_brokers += ':' + kafka_broker_port - -metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] - -local_grok_patterns_dir = format("{metron_home}/patterns") -hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") - -# for create_hdfs_directory -security_enabled = config['configurations']['cluster-env']['security_enabled'] -hostname = config["hostname"] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) -hdfs_site = config['configurations']['hdfs-site'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] -dfs_type = default("/commandParams/dfs_type", "") - -# create partial functions with common arguments for every HdfsResource call -# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled=security_enabled, - keytab=hdfs_user_keytab, - kinit_path_local=kinit_path_local, - hadoop_bin_dir=hadoop_bin_dir, - hadoop_conf_dir=hadoop_conf_dir, - principal_name=hdfs_principal_name, - hdfs_site=hdfs_site, - default_fs=default_fs, - immutable_paths=get_not_managed_resources(), - dfs_type=dfs_type -) +parsers = "bro,yaf,snort" +configured_flag_file = "" diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py index 1290627567..4a5b910918 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py @@ -27,7 +27,7 @@ class ParserMaster(Script): def get_component_name(self): # TODO add this at some point - currently will cause problems with hdp-select - #return "parser-master" + # return "parser-master" pass def install(self, env): @@ -66,42 +66,12 @@ def status(self, env): if not commands.topologies_running(): raise ComponentIsNotRunning() -# from params import status_params -# env.set_params(params) -# commands = Commands(params) -# if not commands.topologies_running(): -# raise ComponentIsNotRunning() - -# from params import status_params -# env.set_params(status_params) -# commands = Commands(params) -# if not commands.topologies_running(): -# raise ComponentIsNotRunning() - def restart(self, env): from params import params env.set_params(params) commands = Commands(params) commands.restart_parser_topologies() - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - - def zookeeperbuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.init_parser_config() - - def starttopologies(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = Commands(params) - commands.start_parser_topologies() - if __name__ == "__main__": ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f4735c2c1f3b7fb3eec8f58a9b6d9a631ccb277 GIT binary patch literal 3700 zcmcgu&2k&Z5$**@kXn)=_2)!!;u!5>^N*N--2t^T8b|KMXm5eN~18~8` zF0?Zcg%WW{DLJPq50FdVA+M8L4!NdsOeJ6UE+N`EJ0N#ZNOX+Rp|C`;L&uop^Aa7GF6L!AE|V_NXo*J47n2I<6?*6J|MBVt z@e1ix3d{5k+uchuF>o(((9zxC|qXBA3vw#4bl#cu2Kk@m}6;$SDCm;14vt< z<1IRZiof~#DEXY0^ffwlDf=5a$JfcxWg6X}cSKION~4=B080J(apKU|AvD^e(JdOl zESG*y%=hB>1Gd+%vMbELN<{9HsIL>FLuFHCPqb=Gf<9h;I7qRrWS-a7pH<@ruA4S*3^{pwHDd!!%l0bsRmK3-7qrw zCLDbh*%L5CMxCYEsTx2|5QY&S8pJ9}Kp2Y(Zl;Gp7DmYshfZc$G(52?Jxg?EPNE4; z>aYd=-h~<_mnsLszF9gg)W2kD;edPUIi#?e4{F=03I_L1v2^F|FJ$XDn5iVSYHHXV zXv=bye$&?z3oW72coIiJ(${WbkyeEWsP+PfvxKx~126|-fl3GBnSxCP)^!zrC)Q5B ztk=)Z&T4@Kq?TsGdYn7g)StkOZNGURs=Ds8B-X~LOuw2&8NAt>sbB&{`auus#le|k ze{wt|$T<5fgKv}JJ!Oh0e=8b!fZ0PATEW!I;ewBXMBQols^#BNj~afjRJ<$CQj|VV)3L*o?m5U`>fVV@$BGj{()ggbY7y}mx|_PIxlmbyN!Zy8hB;*ZPH+A~=O|7negr9hqWLm^pD%;ALJ`zi=34gf zyh5iH%Kj`QoKlchm{jnON|zQP+)3y=0>FK6iKmcxun=FpPp~@3`8$)Ckk{_qWny2Yx8V6Q~!h5P`##X?F#MDVB(@c+nUr|$djE)DE(Raf8 z7t+i@FtOJ_EJIubRVW_@kyICGpPLPwUEdV$^gb`AB2;ae|sy_`TdAz6^#Pvf)? z&-gjUKLK5mj+_nWXU?W`!*QKYoe%K{gxU0EJObfuWB@}LKm%}hUv6E%7!YRIt`Q*1 z3`;=gPXwLIn1QWAzzK&?r`9;9)co^c(2yFFKYUe(-H zVJgkmpp2WrOjEJl0aBxiJ+85F{urr!;t!9b((_w#c{ zX_N?aevk3k`&GuZ$A!h?y`mOdZbYpjP*M&SYMCdvGVznMOnWiUxdL(6zFp3QSJqhQ zB91wk&zG}yaJRFT?}B5g7`b^j@Quc-`!Fnzq|iMsrBWf$YH=OwpXBVkU8MQ2JsEmG z#)Gc)9OVEp+h8^yVIb#Y=jYBR&U)#(^TSf5 z^r3SdMW=$Vit}kHFX$ZYu3)~~m0am|$7wi?b+_xWcf2n!ytnatHq_%p`zap7v96S^ ztW|%2iJ+j{4bwi9WJux$CD#-LoI8Wfg1+vd^9c+lfjyBOVU$U$WAq0-BlO%}zRec) z`a53hX{Wo}^1M4NiV@>?8aw+w(*!Xn`ptHyyVrWslmk*UHaUJUk#)ZR6kZc`p1$b% zhtIs7rpLD!AvNXZjBK_{(n6a6L$-@MwZ|3 + 2.0 From 1bae8d4f946503e7b4e89a9331fa759ea435a326 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Mon, 22 Aug 2016 11:40:48 -0400 Subject: [PATCH 33/82] remove errant file --- .../PARSERS/0.2.0BETA/service_advisor.pyc | Bin 3700 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.pyc deleted file mode 100644 index 9f4735c2c1f3b7fb3eec8f58a9b6d9a631ccb277..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3700 zcmcgu&2k&Z5$**@kXn)=_2)!!;u!5>^N*N--2t^T8b|KMXm5eN~18~8` zF0?Zcg%WW{DLJPq50FdVA+M8L4!NdsOeJ6UE+N`EJ0N#ZNOX+Rp|C`;L&uop^Aa7GF6L!AE|V_NXo*J47n2I<6?*6J|MBVt z@e1ix3d{5k+uchuF>o(((9zxC|qXBA3vw#4bl#cu2Kk@m}6;$SDCm;14vt< z<1IRZiof~#DEXY0^ffwlDf=5a$JfcxWg6X}cSKION~4=B080J(apKU|AvD^e(JdOl zESG*y%=hB>1Gd+%vMbELN<{9HsIL>FLuFHCPqb=Gf<9h;I7qRrWS-a7pH<@ruA4S*3^{pwHDd!!%l0bsRmK3-7qrw zCLDbh*%L5CMxCYEsTx2|5QY&S8pJ9}Kp2Y(Zl;Gp7DmYshfZc$G(52?Jxg?EPNE4; z>aYd=-h~<_mnsLszF9gg)W2kD;edPUIi#?e4{F=03I_L1v2^F|FJ$XDn5iVSYHHXV zXv=bye$&?z3oW72coIiJ(${WbkyeEWsP+PfvxKx~126|-fl3GBnSxCP)^!zrC)Q5B ztk=)Z&T4@Kq?TsGdYn7g)StkOZNGURs=Ds8B-X~LOuw2&8NAt>sbB&{`auus#le|k ze{wt|$T<5fgKv}JJ!Oh0e=8b!fZ0PATEW!I;ewBXMBQols^#BNj~afjRJ<$CQj|VV)3L*o?m5U`>fVV@$BGj{()ggbY7y}mx|_PIxlmbyN!Zy8hB;*ZPH+A~=O|7negr9hqWLm^pD%;ALJ`zi=34gf zyh5iH%Kj`QoKlchm{jnON|zQP+)3y=0>FK6iKmcxun=FpPp~@3`8$)Ckk{_qWny2Yx8V6Q~!h5P`##X?F#MDVB(@c+nUr|$djE)DE(Raf8 z7t+i@FtOJ_EJIubRVW_@kyICGpPLPwUEdV$^gb`AB2;ae|sy_`TdAz6^#Pvf)? z&-gjUKLK5mj+_nWXU?W`!*QKYoe%K{gxU0EJObfuWB@}LKm%}hUv6E%7!YRIt`Q*1 z3`;=gPXwLIn1QWAzzK&?r`9;9)co^c(2yFFKYUe(-H zVJgkmpp2WrOjEJl0aBxiJ+85F{urr!;t!9b((_w#c{ zX_N?aevk3k`&GuZ$A!h?y`mOdZbYpjP*M&SYMCdvGVznMOnWiUxdL(6zFp3QSJqhQ zB91wk&zG}yaJRFT?}B5g7`b^j@Quc-`!Fnzq|iMsrBWf$YH=OwpXBVkU8MQ2JsEmG z#)Gc)9OVEp+h8^yVIb#Y=jYBR&U)#(^TSf5 z^r3SdMW=$Vit}kHFX$ZYu3)~~m0am|$7wi?b+_xWcf2n!ytnatHq_%p`zap7v96S^ ztW|%2iJ+j{4bwi9WJux$CD#-LoI8Wfg1+vd^9c+lfjyBOVU$U$WAq0-BlO%}zRec) z`a53hX{Wo}^1M4NiV@>?8aw+w(*!Xn`ptHyyVrWslmk*UHaUJUk#)ZR6kZc`p1$b% zhtIs7rpLD!AvNXZjBK_{(n6a6L$-@MwZ|3 Date: Fri, 26 Aug 2016 10:12:57 -0400 Subject: [PATCH 34/82] METRON-383 empty commit for rebase with master to fix pre-existing commit problem in master From 00a0468b51482e43a0c1e31a796a07ceaaedc264 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 29 Aug 2016 17:33:08 -0400 Subject: [PATCH 35/82] Working base install with global config --- .../INDEXING/0.2.0BETA/metainfo.xml | 26 -- .../0.2.0BETA/role_command_order.json | 9 - .../METRON/0.2.0BETA/metainfo.xml | 4 +- .../METRON/0.2.0BETA/role_command_order.json | 6 +- .../PARSERS/0.2.0BETA/metainfo.xml | 27 -- .../PARSERS/0.2.0BETA/role_command_order.json | 17 -- .../configuration/metron-indexing.xml | 48 ---- .../INDEXING/0.2.0BETA/metainfo.xml | 105 ------- .../0.2.0BETA/package/scripts/params.py | 29 -- .../0.2.0BETA/package/scripts/params_linux.py | 66 ----- .../package/scripts/params_windows.py | 17 -- .../package/scripts/service_check.py | 37 --- .../INDEXING/0.2.0BETA/service_advisor.py | 65 ----- .../0.2.0BETA/configuration/metron-env.xml | 64 +++-- .../METRON/0.2.0BETA/metainfo.xml | 269 ++++++++++-------- .../0.2.0BETA/package/scripts/commands.py | 253 ---------------- .../package/scripts/indexing_commands.py} | 56 ++-- .../package/scripts/indexing_master.py | 68 +++-- .../package/scripts/metron_master.py | 68 ----- .../0.2.0BETA/package/scripts/params.py | 29 -- .../package/scripts/params/__init__.py | 0 .../package/scripts/params/params.py | 0 .../package/scripts/params/params_linux.py | 20 +- .../package/scripts/params/params_windows.py | 0 .../0.2.0BETA/package/scripts/params_linux.py | 113 -------- .../package/scripts/params_windows.py | 17 -- .../package/scripts/parser_commands.py} | 69 ++--- .../package/scripts/parser_master.py | 48 ++-- .../package/scripts/service_check.py | 13 +- .../package/templates/metron-global.json | 1 - .../0.2.0BETA/quicklinks/quicklinks.json | 0 .../configuration/metron-parsers.xml | 57 ---- .../PARSERS/0.2.0BETA/metainfo.xml | 108 ------- .../package/scripts/params/status_params.py | 22 -- .../package/scripts/service_check.py | 40 --- .../package/templates/metron-global.json | 1 - .../0.2.0BETA/quicklinks/quicklinks.json | 28 -- .../PARSERS/0.2.0BETA/service_advisor.py | 68 ----- .../src/main/resources/mpack.json | 22 +- 39 files changed, 383 insertions(+), 1507 deletions(-) delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/role_command_order.json delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py mode change 100755 => 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml mode change 100755 => 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/commands.py rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{INDEXING/0.2.0BETA/package/scripts/commands.py => METRON/0.2.0BETA/package/scripts/indexing_commands.py} (75%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{INDEXING => METRON}/0.2.0BETA/package/scripts/indexing_master.py (52%) delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_master.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params.py rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS => METRON}/0.2.0BETA/package/scripts/params/__init__.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS => METRON}/0.2.0BETA/package/scripts/params/params.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS => METRON}/0.2.0BETA/package/scripts/params/params_linux.py (84%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS => METRON}/0.2.0BETA/package/scripts/params/params_windows.py (100%) delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_linux.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_windows.py rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS/0.2.0BETA/package/scripts/commands.py => METRON/0.2.0BETA/package/scripts/parser_commands.py} (83%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/{PARSERS => METRON}/0.2.0BETA/package/scripts/parser_master.py (64%) delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/metron-global.json mode change 100644 => 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json delete mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/metainfo.xml deleted file mode 100755 index e406780977..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/metainfo.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - 2.0 - - - INDEXING - 0.2.0BETA - common-services/INDEXING/0.2.0BETA - - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/role_command_order.json deleted file mode 100755 index f58c59c322..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/INDEXING/0.2.0BETA/role_command_order.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], - "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"] - } -} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml index e406780977..a3c984c612 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml @@ -18,9 +18,9 @@ 2.0 - INDEXING + METRON 0.2.0BETA - common-services/INDEXING/0.2.0BETA + common-services/METRON/0.2.0BETA diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json index f58c59c322..871062c133 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json @@ -3,7 +3,9 @@ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", "general_deps" : { "_comment" : "dependencies for all cases", - "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], - "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"] + "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], + "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], + "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], + "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"] } } diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml deleted file mode 100755 index e0292cfabf..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/metainfo.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - 2.0 - - - PARSERS - 0.2.0BETA - common-services/PARSERS/0.2.0BETA - - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json deleted file mode 100755 index c8616469b6..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/PARSERS/0.2.0BETA/role_command_order.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "_comment": "Record format:", - "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps": { - "_comment": "dependencies for Metron Parsers", - "PARSER_MASTER-START": [ - "NAMENODE-START", - "ZOOKEEPER_SERVER-START", - "KAFKA_BROKER-START", - "STORM_REST_API-START" - ], - "_comment": "_SERVICE_CHECK-SERVICE_CHECK", - "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [ - "PARSER_MASTER-START" - ] - } -} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml deleted file mode 100755 index 3a1c8f79df..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - metron_home - /usr/metron/0.2.0BETA - Metron home directory - Metron home - - - metron_apps_hdfs_dir - /apps/metron - Metron apps HDFS dir - Metron apps HDFS dir - - - metron_user - metron - USER - The user for Metron - Metron User - - - metron_group - metron - The group for Metron - - - metron_indexing_topology - indexing - The Storm topology name for Indexing - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml deleted file mode 100755 index d2dce5ac3f..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - 2.0 - - - INDEXING - Indexing - Indexing - 0.2.0BETA - - - - INDEXING_MASTER - Indexing Master-Eligible Node - MASTER - 1 - true - - - HDFS/HDFS_CLIENT - host - - true - - - - ZOOKEEPER/ZOOKEEPER_SERVER - cluster - - true - - - - KAFKA/KAFKA_BROKER - cluster - - true - - - - - - PYTHON - 600 - - - - - - - any - - - metron-common - - - metron-indexing - - - metron-elasticsearch - - - - - - - - PYTHON - 300 - - - - HDFS - KAFKA - STORM - ZOOKEEPER - - - - metron-indexing - - - - - quicklinks.json - true - - - - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py deleted file mode 100755 index 6440005faf..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from ambari_commons import OSCheck -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions.expect import expect - -if OSCheck.is_windows_family(): - pass -else: - from params_linux import * - -java_home = config['hostLevelParams']['java_home'] -java_version = expect("/hostLevelParams/java_version", int) - -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py deleted file mode 100755 index ee5f546c25..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import os - -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.default import default -from resource_management.libraries.script import Script - -# Server configurations -config = Script.get_config() - -hostname = config['hostname'] -metron_home = config['configurations']['metron-indexing']['metron_home'] -metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology'] -yum_repo_type = 'local' - -# Hadoop params -hadoop_home_dir = stack_select.get_hadoop_dir("home") -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hadoop_conf_dir = conf_select.get_hadoop_conf_dir() - -# Zookeeper -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -has_zk_host = not len(zk_hosts) == 0 -zookeeper_quorum = None -if has_zk_host: - if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: - zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] - else: - zookeeper_clientPort = '2181' - zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) - # last port config - zookeeper_quorum += ':' + zookeeper_clientPort - -# Kafka -stack_root = Script.get_stack_root() -kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin") -metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention'] - -kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) -has_kafka_host = not len(kafka_hosts) == 0 -kafka_brokers = None -if has_kafka_host: - if 'port' in config['configurations']['kafka-broker']: - kafka_broker_port = config['configurations']['kafka-broker']['port'] - else: - kafka_broker_port = '6667' - kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) - kafka_brokers += ':' + kafka_broker_port diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py deleted file mode 100755 index b5828d227c..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -raise NotImplementedError diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py deleted file mode 100755 index c02d695919..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from __future__ import print_function - -from resource_management.libraries.script import Script - -from commands import Commands - - -class ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - commands = Commands(params) - if commands.is_topology_active(): - exit(0) - else: - exit(1) - - -if __name__ == "__main__": - ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py deleted file mode 100755 index a38c182f43..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/service_advisor.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env ambari-python-wrap -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import imp -import os -import traceback - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') -PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') - -try: - with open(PARENT_FILE, 'rb') as fp: - service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) -except Exception as e: - traceback.print_exc() - print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) - - -class INDEXING020BETAServiceAdvisor(service_advisor.ServiceAdvisor): - # colocate Metron Parser Master with KAFKA_BROKERs - def colocateService(self, hostsComponentsMap, serviceComponents): - indexingMasterComponent = [component for component in serviceComponents if - component["StackServiceComponents"]["component_name"] == "INDEXING_MASTER"] - indexingMasterComponent = indexingMasterComponent[0] - if not self.isComponentHostsPopulated(indexingMasterComponent): - for hostName in hostsComponentsMap.keys(): - hostComponents = hostsComponentsMap[hostName] - if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "INDEXING_MASTER"} not in hostComponents: - hostsComponentsMap[hostName].append({"name": "INDEXING_MASTER"}) - if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "INDEXING_MASTER"} in hostComponents: - hostsComponentsMap[hostName].remove({"name": "INDEXING_MASTER"}) - - def getServiceComponentLayoutValidations(self, services, hosts): - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - - indexingHosts = self.getHosts(componentsList, "INDEXING_MASTER") - brokerHosts = self.getHosts(componentsList, "KAFKA_BROKER") - - items = [] - - mismatchHosts = sorted(set(indexingHosts).symmetric_difference(set(brokerHosts))) - if len(mismatchHosts) > 0: - hostsString = ', '.join(mismatchHosts) - message = "Metron Indexing Master must be installed on Kafka Brokers. " \ - "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( - len(mismatchHosts), hostsString) - items.append( - {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'INDEXING_MASTER'}) - - return items diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml old mode 100755 new mode 100644 index 25312a77a8..4bda11c6b3 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -8,7 +8,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,23 +24,18 @@ Metron home directory Metron home - - parsers - bro,snort,yaf - Metron parsers to deploy - Metron parsers - - - metron_indexing_topology - indexing - The Storm topology name for Indexing - metron_apps_hdfs_dir /apps/metron Metron apps HDFS dir Metron apps HDFS dir + + metron_zookeeper_config_dir + config/zookeeper + Metron Zookeeper config dir. Relative path to Metron home. + Metron Zookeeper config dir + metron_user metron @@ -49,13 +46,48 @@ metron_group metron + GROUP The group for Metron - metron_zookeeper_config_dir - config/zookeeper - Metron Zookeeper config dir. Relative path to Metron home. - Metron Zookeeper config dir + metron_topic_retention + Kafka Retention in GB + 10 + + + parsers + bro,snort,yaf + Metron parsers to deploy + Metron parsers + + + metron_indexing_topology + indexing + The Storm topology name for Indexing + + + es_cluster_name + metron + Name of Elasticsearch Cluster + + + es_url + + Comma delimited list of Elasticsearch URLs. (eshost1:9200,eshost2:9200) + + + content + global.json template + This is the jinja template for global.json file + +{ +"es.clustername": "{{ es_cluster_name }}", +"es.ip": "[{{ es_url }}]", +"es.date.format": "yyyy.MM.dd.HH" +} + + + content + - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml old mode 100755 new mode 100644 index 50ae1d2f77..6f8a46109f --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -6,7 +6,9 @@ The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,131 +16,152 @@ limitations under the License. --> - 2.0 - - - INDEXING - Indexing - Indexing - 0.2.0BETA - - - PARSER_MASTER - Metron Parser Topologies - MASTER - 1 - true - - - HDFS/HDFS_CLIENT - host - - true - - - - ZOOKEEPER/ZOOKEEPER_SERVER - cluster - - true - - - - KAFKA/KAFKA_BROKER - cluster - - true - - - - - - PYTHON - 600 - - - metron-parsers - - - - INDEXING_MASTER - Indexing Master - MASTER - 1 - true - - - HDFS/HDFS_CLIENT - host - - true - - - - ZOOKEEPER/ZOOKEEPER_SERVER - cluster - - true - - - - KAFKA/KAFKA_BROKER - cluster - - true - - - - - - PYTHON - 600 - - - - - - any - - - metron-common - - - metron-parsers - - - metron-indexing - - - metron-elasticsearch - - - - + 2.0 + + + METRON + Metron + A scalable advanced security analytics framework built with on Hadoop + 0.2.0BETA + + + + METRON_PARSERS + Metron Parsers + MASTER + 1 + true + false + + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + STORM/SUPERVISOR + host + + true + + + + KAFKA/KAFKA_BROKER + host + + true + + + + + + PYTHON + + + metron-parsers + + + + + METRON_INDEXING + Metron Indexing + MASTER + 1 + true + false + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + STORM/SUPERVISOR + host + + true + + + + KAFKA/KAFKA_BROKER + host + + true + + + + + + PYTHON + + + metron-indexing + + + + + - - - PYTHON - 300 - + + + any + + + metron-common + + + metron-parsers + + + metron-indexing + + + metron-elasticsearch + + + + - - HDFS - KAFKA - STORM - ZOOKEEPER - + + + PYTHON + 300 + - - metron-indexing - + + HDFS + KAFKA + STORM + ZOOKEEPER + - - - quicklinks.json - true - - - - + + hive-log4j + hive-exec-log4j + hive-env + hivemetastore-site.xml + webhcat-site + webhcat-env + ranger-hive-plugin-properties + ranger-hive-audit + ranger-hive-policymgr-ssl + ranger-hive-security + mapred-site + application.properties + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/commands.py deleted file mode 100755 index 9842194f76..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/commands.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import subprocess -import time -import os - -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File -from resource_management.core.source import Template -from resource_management.libraries.functions import format - -# Wrap major operations and functionality in this class -class Commands: - __params = None - __indexing = None - __parser_list = None - __configured = False - - def __init__(self, params): - if params is None: - raise ValueError("params argument is required for initialization") - self.__params = params - self.__indexing = params.metron_indexing_topology - self.__parser_list = self.__get_parsers(params) - self.__configured = os.path.isfile(self.__params.configured_flag_file) - - def setup_repo(self): - def local_repo(): - Logger.info("Setting up local repo") - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - Execute("echo \"[METRON-0.2.0BETA]\n" - "name=Metron 0.2.0BETA packages\n" - "baseurl=file:///localrepo\n" - "gpgcheck=0\n" - "enabled=1\" > /etc/yum.repos.d/local.repo") - - def remote_repo(): - print('Using remote repo') - - yum_repo_types = { - 'local': local_repo, - 'remote': remote_repo - } - repo_type = self.__params.yum_repo_type - if repo_type in yum_repo_types: - yum_repo_types[repo_type]() - else: - raise ValueError("Unsupported repo type '{}'".format(repo_type)) - - def init_kafka_topics(self): - Logger.info('Creating Kafka topics') - command_template = """{}/kafka-topics.sh \ - --zookeeper {} \ - --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""" - num_partitions = 1 - replication_factor = 1 - retention_gigabytes = 10 - retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 - Logger.info("Creating topics for indexing") - - Logger.info("Creating topic'{}'".format(self.__indexing)) - Execute(command_template.format(self.__params.kafka_bin_dir, - self.__params.zookeeper_quorum, - self.__indexing, - num_partitions, - replication_factor, - retention_bytes)) - Logger.info("Done creating Kafka topics") - - #Parsers - def init_parsers(self): - Logger.info( - "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, - self.__params.metron_apps_dir)) - self.__params.HdfsResource(self.__params.metron_apps_dir, - type="directory", - action="create_on_execute", - owner=self.__params.metron_user, - mode=0775, - source=self.__params.local_grok_patterns_dir) - - Logger.info("Creating global.json file") - File(self.__params.metron_zookeeper_config_path + '/global.json', - content=Template("metron-global.json"), - owner=self.__params.metron_user, - mode=0775) - Logger.info("Done initializing parser configuration") - - def get_parser_list(self): - return self.__parser_list - - def init_parser_config(self): - Logger.info('Loading parser config into ZooKeeper') - Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), - path=format("{java_home}/bin") - ) - - def start_parser_topologies(self): - Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) - start_cmd_template = """{}/bin/start_parser_topology.sh \ - -k {} \ - -z {} \ - -s {}""" - for parser in self.get_parser_list(): - Logger.info('Starting ' + parser) - Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, - self.__params.zookeeper_quorum, parser)) - - Logger.info('Finished starting parser topologies') - - def stop_parser_topologies(self): - Logger.info('Stopping parsers') - for parser in self.get_parser_list(): - Logger.info('Stopping ' + parser) - stop_cmd = 'storm kill ' + parser - Execute(stop_cmd) - Logger.info('Done stopping parser topologies') - - def restart_parser_topologies(self): - Logger.info('Restarting the parser topologies') - self.stop_parser_topologies() - attempt_count = 0 - while self.topologies_exist(): - if attempt_count > 2: - raise Exception("Unable to kill topologies") - attempt_count += 1 - time.sleep(10) - self.start_parser_topologies() - Logger.info('Done restarting the parser topologies') - - def topologies_exist(self): - cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = cmd_open.communicate() - stdout_lines = stdout.splitlines() - if stdout_lines: - status_lines = self.__get_status_lines(stdout_lines) - for parser in self.get_parser_list(): - for line in status_lines: - items = re.sub('[\s]+', ' ', line).split() - if items and items[0] == parser: - return True - return False - - def topologies_running(self): - cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = cmd_open.communicate() - stdout_lines = stdout.splitlines() - all_running = False - if stdout_lines: - all_running = True - status_lines = self.__get_status_lines(stdout_lines) - for parser in self.get_parser_list(): - parser_found = False - is_running = False - for line in status_lines: - items = re.sub('[\s]+', ' ', line).split() - if items and items[0] == parser: - status = items[1] - parser_found = True - is_running = self.__is_running(status) - all_running &= parser_found and is_running - return all_running - - def __get_status_lines(self, lines): - status_lines = [] - do_stat = False - skipped = 0 - for line in lines: - if line.startswith("Topology_name"): - do_stat = True - if do_stat and skipped == 2: - status_lines += [line] - elif do_stat: - skipped += 1 - return status_lines - - def __is_running(self, status): - return status in ['ACTIVE', 'REBALANCING'] - - #Indexing - def start_indexing_topology(self): - Logger.info("Starting Metron indexing topology: {}".format(self.__indexing)) - start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \ - -s {} \ - -z {}""" - Logger.info('Starting ' + self.__indexing) - Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum)) - - Logger.info('Finished starting indexing topology') - - def stop_indexing_topology(self): - Logger.info('Stopping ' + self.__indexing) - stop_cmd = 'storm kill ' + self.__indexing - Execute(stop_cmd) - Logger.info('Done stopping indexing topologies') - - def restart_indexing_topology(self): - Logger.info('Restarting the indexing topologies') - self.stop_indexing_topology() - - # Wait for old topology to be cleaned up by Storm, before starting again. - retries = 0 - topology_active = self.is_topology_active() - while topology_active and retries < 3: - Logger.info('Existing topology still active. Will wait and retry') - time.sleep(40) - topology_active = self.is_topology_active() - retries += 1 - - if not topology_active: - self.start_indexing_topology() - Logger.info('Done restarting the indexing topologies') - else: - Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') - - def is_topology_active(self): - cmd_retrieve = "storm list | grep 'indexing'" - proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - Logger.info("Retrieval response is: %s" % stdout) - Logger.warning("Error response is: %s" % stderr) - - fields = stdout.split() - if len(fields) < 2: - Logger.warning("Indexing topology is not running") - return False - - # Get the second column, which is status. We already know first column is indexing) - status = stdout.split()[1] - running_status_set = {'ACTIVE', 'REBALANCING'} - return status in running_status_set diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py similarity index 75% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index b47517f185..fbab275ef0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -19,11 +19,11 @@ import time from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute - +from resource_management.core.resources.system import Execute, File +from resource_management.libraries.functions import format # Wrap major operations and functionality in this class -class Commands: +class IndexingCommands: __params = None __indexing = None @@ -69,7 +69,7 @@ def init_kafka_topics(self): --config retention.bytes={}""" num_partitions = 1 replication_factor = 1 - retention_gigabytes = 10 + retention_gigabytes = int(self.__params.metron_topic_retention) retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for indexing") @@ -117,19 +117,37 @@ def restart_indexing_topology(self): else: Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def init_config(self): + Logger.info('Loading config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) + def is_topology_active(self): - cmd_retrieve = "storm list | grep 'indexing'" - proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - Logger.info("Retrieval response is: %s" % stdout) - Logger.warning("Error response is: %s" % stderr) - - fields = stdout.split() - if len(fields) < 2: - Logger.warning("Indexing topology is not running") - return False - - # Get the second column, which is status. We already know first column is indexing) - status = stdout.split()[1] - running_status_set = {'ACTIVE', 'REBALANCING'} - return status in running_status_set + # cmd_retrieve = "storm list | grep 'indexing'" + # proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + # (stdout, stderr) = proc.communicate() + # Logger.info("Retrieval response is: %s" % stdout) + # Logger.warning("Error response is: %s" % stderr) + # + # fields = stdout.split() + # if len(fields) < 2: + # Logger.warning("Indexing topology is not running") + # return False + # + # # Get the second column, which is status. We already know first column is indexing) + # status = stdout.split()[1] + # running_status_set = {'ACTIVE', 'REBALANCING'} + # return status in running_status_set + # + return True diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py similarity index 52% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index e233379bb1..84b71efee9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -17,52 +17,76 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script - -from commands import Commands +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from indexing_commands import IndexingCommands class Indexing(Script): + + __configured = False + def install(self, env): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = IndexingCommands(params) commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) + def configure(self, env): + + from params import params + env.set_params(params) + + Logger.info("Configure Metron global.json") + + directories = [params.metron_zookeeper_config_path] + Directory(directories, + # recursive=True, + mode=0755, + owner=params.metron_user, + group=params.metron_group + ) + + File("{}/global.json".format(params.metron_zookeeper_config_path), + owner=params.metron_user, + content=InlineTemplate(params.global_json_template) + ) + commands = IndexingCommands(params) + commands.init_config() + def start(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - commands = Commands(params) + self.configure(env) + commands = IndexingCommands(params) commands.init_kafka_topics() commands.start_indexing_topology() def stop(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = IndexingCommands(params) commands.stop_indexing_topology() def status(self, env): - import params - env.set_params(params) - commands = Commands(params) - - if not commands.is_topology_active(): - raise ComponentIsNotRunning() + # from params import params + # env.set_params(params) + # + # commands = IndexingCommands(params) + # + # if not commands.is_topology_active(): + raise ComponentIsNotRunning() def restart(self, env): - import params + from params import params env.set_params(params) - commands = Commands(params) + self.configure(env) + commands = IndexingCommands(params) commands.restart_indexing_topology() - def kafkabuild(self, env, upgrade_type=None): - import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - if __name__ == "__main__": Indexing().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_master.py deleted file mode 100755 index e233379bb1..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_master.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -from resource_management.libraries.script import Script - -from commands import Commands - - -class Indexing(Script): - def install(self, env): - import params - env.set_params(params) - commands = Commands(params) - commands.setup_repo() - Logger.info('Install RPM packages') - self.install_packages(env) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - commands.start_indexing_topology() - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - commands = Commands(params) - commands.stop_indexing_topology() - - def status(self, env): - import params - env.set_params(params) - commands = Commands(params) - - if not commands.is_topology_active(): - raise ComponentIsNotRunning() - - def restart(self, env): - import params - env.set_params(params) - commands = Commands(params) - commands.restart_indexing_topology() - - def kafkabuild(self, env, upgrade_type=None): - import params - env.set_params(params) - commands = Commands(params) - commands.init_kafka_topics() - - -if __name__ == "__main__": - Indexing().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params.py deleted file mode 100755 index 6440005faf..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from ambari_commons import OSCheck -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions.expect import expect - -if OSCheck.is_windows_family(): - pass -else: - from params_linux import * - -java_home = config['hostLevelParams']['java_home'] -java_version = expect("/hostLevelParams/java_version", int) - -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py similarity index 84% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index de7dfc9ff2..5dd5ebfd0a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -32,17 +32,18 @@ # server configurations config = Script.get_config() - hostname = config['hostname'] -#print(config['configurations']) -metron_home = config['configurations']['metron-parsers']['metron_home'] -parsers = config['configurations']['metron-parsers']['parsers'] -metron_user = config['configurations']['metron-parsers']['metron_user'] -metron_group = config['configurations']['metron-parsers']['metron_group'] -metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir +metron_home = config['configurations']['metron-env']['metron_home'] +parsers = config['configurations']['metron-env']['parsers'] +metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] +metron_user = config['configurations']['metron-env']['metron_user'] +metron_group = config['configurations']['metron-env']['metron_group'] +metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' +global_json_template = config['configurations']['metron-env']['content'] +es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] +es_url = config['configurations']['metron-env']['es_url'] yum_repo_type = 'local' # hadoop params @@ -78,7 +79,8 @@ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) kafka_brokers += ':' + kafka_broker_port -metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] +metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir'] +metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention'] local_grok_patterns_dir = format("{metron_home}/patterns") hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_linux.py deleted file mode 100755 index a7336cdb23..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_linux.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import functools -import os - -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.default import default -from resource_management.libraries.script import Script -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources - -# Server configurations -config = Script.get_config() - -hostname = config['hostname'] -metron_home = config['configurations']['metron-indexing']['metron_home'] -parsers = config['configurations']['metron-parsers']['parsers'] -metron_user = config['configurations']['metron-parsers']['metron_user'] -metron_group = config['configurations']['metron-parsers']['metron_group'] -metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology'] -yum_repo_type = 'local' - - -# Hadoop params -hadoop_home_dir = stack_select.get_hadoop_dir("home") -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hadoop_conf_dir = conf_select.get_hadoop_conf_dir() - - -# Zookeeper -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -has_zk_host = not len(zk_hosts) == 0 -zookeeper_quorum = None -if has_zk_host: - if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: - zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] - else: - zookeeper_clientPort = '2181' - zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) - # last port config - zookeeper_quorum += ':' + zookeeper_clientPort - -# Kafka -stack_root = Script.get_stack_root() -kafka_home = os.path.join(stack_root, "current", "kafka-broker") -kafka_bin_dir = os.path.join(kafka_home, "bin") -metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention'] - -kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) -has_kafka_host = not len(kafka_hosts) == 0 -kafka_brokers = None -if has_kafka_host: - if 'port' in config['configurations']['kafka-broker']: - kafka_broker_port = config['configurations']['kafka-broker']['port'] - else: - kafka_broker_port = '6667' - kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) - kafka_brokers += ':' + kafka_broker_port - -metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir'] - -local_grok_patterns_dir = format("{metron_home}/patterns") -hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns") - -# for create_hdfs_directory -security_enabled = config['configurations']['cluster-env']['security_enabled'] -hostname = config["hostname"] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) -hdfs_site = config['configurations']['hdfs-site'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] -dfs_type = default("/commandParams/dfs_type", "") - -# create partial functions with common arguments for every HdfsResource call -# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled=security_enabled, - keytab=hdfs_user_keytab, - kinit_path_local=kinit_path_local, - hadoop_bin_dir=hadoop_bin_dir, - hadoop_conf_dir=hadoop_conf_dir, - principal_name=hdfs_principal_name, - hdfs_site=hdfs_site, - default_fs=default_fs, - immutable_paths=get_not_managed_resources(), - dfs_type=dfs_type -) \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_windows.py deleted file mode 100755 index b5828d227c..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params_windows.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -raise NotImplementedError diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py similarity index 83% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index 6b33e55a55..9ba740ce0f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -30,7 +30,7 @@ # Wrap major operations and functionality in this class -class Commands: +class ParserCommands: __params = None __parser_list = None __configured = False @@ -55,14 +55,6 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) - # Possible storm topology status states - # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html - class StormStatus: - ACTIVE = "ACTIVE" - INACTIVE = "INACTIVE" - KILLED = "KILLED" - REBALANCING = "REBALANCING" - def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, @@ -74,11 +66,6 @@ def init_parsers(self): mode=0775, source=self.__params.local_grok_patterns_dir) - Logger.info("Creating global.json file") - File(self.__params.metron_zookeeper_config_path + '/global.json', - content=Template("metron-global.json"), - owner=self.__params.metron_user, - mode=0775) Logger.info("Done initializing parser configuration") def get_parser_list(self): @@ -145,13 +132,6 @@ def init_kafka_topics(self): retention_bytes)) Logger.info("Done creating Kafka topics") - def init_parser_config(self): - Logger.info('Loading parser config into ZooKeeper') - Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), - path=format("{java_home}/bin") - ) - def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) start_cmd_template = """{}/bin/start_parser_topology.sh \ @@ -185,6 +165,13 @@ def restart_parser_topologies(self): self.start_parser_topologies() Logger.info('Done restarting the parser topologies') + def init_config(self): + Logger.info('Loading config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) + def topologies_exist(self): cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = cmd_open.communicate() @@ -199,24 +186,26 @@ def topologies_exist(self): return False def topologies_running(self): - cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = cmd_open.communicate() - stdout_lines = stdout.splitlines() - all_running = False - if stdout_lines: - all_running = True - status_lines = self.__get_status_lines(stdout_lines) - for parser in self.get_parser_list(): - parser_found = False - is_running = False - for line in status_lines: - items = re.sub('[\s]+', ' ', line).split() - if items and items[0] == parser: - status = items[1] - parser_found = True - is_running = self.__is_running(status) - all_running &= parser_found and is_running - return all_running + #TODO - use Storm Rest Commands + # cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # (stdout, stderr) = cmd_open.communicate() + # stdout_lines = stdout.splitlines() + # all_running = False + # if stdout_lines: + # all_running = True + # status_lines = self.__get_status_lines(stdout_lines) + # for parser in self.get_parser_list(): + # parser_found = False + # is_running = False + # for line in status_lines: + # items = re.sub('[\s]+', ' ', line).split() + # if items and items[0] == parser: + # status = items[1] + # parser_found = True + # is_running = self.__is_running(status) + # all_running &= parser_found and is_running + # return all_running + return True def __get_status_lines(self, lines): status_lines = [] @@ -232,4 +221,4 @@ def __get_status_lines(self, lines): return status_lines def __is_running(self, status): - return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING] + return status in ['ACTIVE', 'REBALANCING'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py similarity index 64% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index 4a5b910918..c145777d47 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -20,8 +20,8 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script - -from commands import Commands +from resource_management.core.resources.system import Directory, File +from parser_commands import ParserCommands class ParserMaster(Script): @@ -33,11 +33,27 @@ def get_component_name(self): def install(self, env): from params import params env.set_params(params) - commands = Commands(params) + commands = ParserCommands(params) commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) + Logger.info('Create Metron Local Config Directory') + directories = [params.metron_zookeeper_config_path] + Directory(directories, + # recursive=True, + mode=0755, + owner=params.metron_user, + group=params.metron_group + ) + + #Default global.json to empty file + File("{}/global.json".format(params.metron_zookeeper_config_path), + owner=params.metron_user, + content="{}" + ) + commands.init_config() + def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) @@ -45,33 +61,31 @@ def configure(self, env, upgrade_type=None, config_dir=None): def start(self, env, upgrade_type=None): from params import params env.set_params(params) - commands = Commands(params) - if not commands.is_configured(): - commands.init_parsers() - commands.init_kafka_topics() - commands.init_parser_config() - commands.set_configured() + commands = ParserCommands(params) + commands.init_parsers() + commands.init_kafka_topics() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): from params import params env.set_params(params) - commands = Commands(params) + commands = ParserCommands(params) commands.stop_parser_topologies() def status(self, env): - from params import status_params - env.set_params(status_params) - commands = Commands(status_params) - if not commands.topologies_running(): - raise ComponentIsNotRunning() + # from params import params + # env.set_params(params) + # + # commands = ParserCommands(params) + # + # if not commands.topologies_running(): + raise ComponentIsNotRunning() def restart(self, env): from params import params env.set_params(params) - commands = Commands(params) + commands = ParserCommands(params) commands.restart_parser_topologies() - if __name__ == "__main__": ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py index c02d695919..947c0b50b4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py @@ -7,27 +7,30 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + """ from __future__ import print_function from resource_management.libraries.script import Script -from commands import Commands +from parser_commands import ParserCommands class ServiceCheck(Script): def service_check(self, env): - import params + from params import params env.set_params(params) - - commands = Commands(params) - if commands.is_topology_active(): + commands = ParserCommands(params) + all_found = commands.topologies_running() + if all_found: exit(0) else: exit(1) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/metron-global.json deleted file mode 100755 index 0967ef424b..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/metron-global.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json old mode 100644 new mode 100755 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml deleted file mode 100755 index 71966eab9f..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - metron_home - /usr/metron/0.2.0BETA - Metron home directory - Metron home - - - parsers - bro,snort,yaf - Metron parsers to deploy - Metron parsers - - - metron_apps_hdfs_dir - /apps/metron - Metron apps HDFS dir - Metron apps HDFS dir - - - metron_zookeeper_config_dir - config/zookeeper - Metron Zookeeper config dir. Relative path to Metron home. - Metron Zookeeper config dir - - - metron_user - metron - USER - The user for Metron - Metron User - - - metron_group - metron - The group for Metron - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml deleted file mode 100755 index f1e5ab7d90..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml +++ /dev/null @@ -1,108 +0,0 @@ - - - - 2.0 - - - PARSERS - Metron Parsers - Parsers - 0.2.0BETA - - - - PARSER_MASTER - Metron Parser Topologies - MASTER - 1 - true - - - HDFS/HDFS_CLIENT - host - - true - - - - ZOOKEEPER/ZOOKEEPER_SERVER - cluster - - true - - - - KAFKA/KAFKA_BROKER - cluster - - true - - - - - - PYTHON - 600 - - - metron-parsers - - - - - - - any - - - metron-common - - - metron-parsers - - - - - - - - PYTHON - 300 - - - - HDFS - KAFKA - STORM - ZOOKEEPER - - - - metron-parsers - - - - - quicklinks.json - true - - - - - - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py deleted file mode 100755 index ee83414f0c..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -parsers = "bro,yaf,snort" -configured_flag_file = "" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py deleted file mode 100755 index 6ef7e4aa14..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from __future__ import print_function - -from resource_management.libraries.script import Script - -from commands import Commands - - -class ServiceCheck(Script): - def service_check(self, env): - from params import params - env.set_params(params) - commands = Commands(params) - all_found = commands.topologies_running() - if all_found: - exit(0) - else: - exit(1) - - -if __name__ == "__main__": - ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json deleted file mode 100755 index 0967ef424b..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json deleted file mode 100755 index ee1b225084..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "storm_ui", - "label": "Storm UI", - "requires_user_name": "false", - "component_name": "STORM_UI_SERVER", - "url":"%@://%@:%@/", - "port":{ - "http_property": "ui.port", - "http_default_port": "8744", - "https_property": "ui.port", - "https_default_port": "8744", - "regex": "^(\\d+)$", - "site": "storm-site" - } - } - ] - } -} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py deleted file mode 100755 index 84b466a353..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env ambari-python-wrap -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import imp -import os -import traceback - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') -PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') - -try: - with open(PARENT_FILE, 'rb') as fp: - service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) -except Exception as e: - traceback.print_exc() - print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) - - -class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor): - # colocate Metron Parser Master with KAFKA_BROKERs - def TODO_colocateService(self, hostsComponentsMap, serviceComponents): - parsersMasterComponent = [component for component in serviceComponents if - component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0] - if not self.isComponentHostsPopulated(parsersMasterComponent): - for hostName in hostsComponentsMap.keys(): - hostComponents = hostsComponentsMap[hostName] - if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents: - hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"}) - if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents: - hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"}) - - def TODO_getServiceComponentLayoutValidations(self, services, hosts): - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] - hostsCount = len(hostsList) - - masterHosts = self.getHosts(componentsList, "PARSER_MASTER") - expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER")) - - items = [] - - mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts))) - if len(mismatchHosts) > 0: - hostsString = ', '.join(mismatchHosts) - message = "Metron Parsers Master must be installed on Kafka Brokers. " \ - "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format( - len(mismatchHosts), hostsString) - items.append( - {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'}) - - return items diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json index 2fbb358697..99dec9b76a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json @@ -4,7 +4,7 @@ "version": "1.0.0.0", "description": "Ambari Management Pack for Apache Metron", "prerequisites": { - "min-ambari-version": "2.2.0.0", + "min-ambari-version": "2.4.0.0", "min-stack-versions": [ { "stack_name": "HDP", @@ -57,6 +57,26 @@ "stack_name" : "HDP", "stack_version" : "2.5" } + + ] + }, + { + "service_name" : "METRON", + "service_version" : "0.2.0BETA", + "applicable_stacks" : [ + { + "stack_name" : "HDP", + "stack_version" : "2.3" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.4" + }, + { + "stack_name" : "HDP", + "stack_version" : "2.5" + } + ] } ] From 119a5ede3e56985b2d00565e86d0874c8279b1cc Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 29 Aug 2016 20:38:36 -0400 Subject: [PATCH 36/82] Service check uses rest commands for storm rather than cli. --- .../METRON/0.2.0BETA/metainfo.xml | 34 +++++++++----- .../package/scripts/indexing_commands.py | 6 --- .../package/scripts/indexing_master.py | 4 +- .../package/scripts/metron_service.py | 45 +++++++++++++++++++ .../package/scripts/params/params_linux.py | 10 +++++ .../package/scripts/parser_commands.py | 43 ++++++------------ .../package/scripts/parser_master.py | 17 ++++--- .../package/scripts/service_check.py | 3 +- 8 files changed, 106 insertions(+), 56 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml index 6f8a46109f..c57b381476 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -67,6 +67,16 @@ PYTHON + + + SERVICECHECKTEST + false + + + PYTHON + + + metron-parsers @@ -149,19 +159,19 @@ - hive-log4j - hive-exec-log4j - hive-env - hivemetastore-site.xml - webhcat-site - webhcat-env - ranger-hive-plugin-properties - ranger-hive-audit - ranger-hive-policymgr-ssl - ranger-hive-security - mapred-site - application.properties + metron-env + storm-env + storm-site + kafka-broker + kafka-env + true + + + quicklinks.json + true + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index fbab275ef0..2e0d5cef29 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -126,12 +126,6 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) - def init_config(self): - Logger.info('Loading config into ZooKeeper') - Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), - path=format("{java_home}/bin") - ) def is_topology_active(self): # cmd_retrieve = "storm list | grep 'indexing'" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index 84b71efee9..899d9d9fe5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -21,7 +21,7 @@ from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate from indexing_commands import IndexingCommands - +import metron_service class Indexing(Script): @@ -55,7 +55,7 @@ def configure(self, env): content=InlineTemplate(params.global_json_template) ) commands = IndexingCommands(params) - commands.init_config() + metron_service.init_config() def start(self, env, upgrade_type=None): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py new file mode 100644 index 0000000000..2f39848d8d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -0,0 +1,45 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.script import Script + +import subprocess +import json + +def init_config(): + Logger.info('Loading config into ZooKeeper') + Execute(format( + "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), + path=format("{java_home}/bin") + ) + +def get_running_topologies(): + Logger.info('Getting Running Storm Topologies from Storm REST Server') + + cmd = format('curl {storm_rest_addr}/api/v1/topology/summary') + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + + stormjson = json.loads(stdout) + topologiesDict = {} + + for topology in stormjson['topologies']: + topologiesDict[topology['name']] = topology['status'] + + return topologiesDict diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 5dd5ebfd0a..0904ddc250 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -67,6 +67,16 @@ # last port config zookeeper_quorum += ':' + zookeeper_clientPort +#Storm +storm_ui_hosts = default("/clusterHostInfo/storm_ui_server_hosts", []) +has_storm_host = not len(storm_ui_hosts) == 0 +if has_storm_host: + if 'storm-site' in config['configurations'] and 'ui.port' in config['configurations']['storm-site']: + storm_ui_port = config['configurations']['storm-site']['ui.port'] + else: + storm_ui_port = '8744' + storm_rest_addr = storm_ui_hosts[0] + ':' + storm_ui_port + # Kafka kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) has_kafka_host = not len(kafka_hosts) == 0 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index 9ba740ce0f..bf1916e32d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -27,7 +27,7 @@ from resource_management.core.resources.system import Execute, File from resource_management.core.source import Template from resource_management.libraries.functions import format - +import metron_service # Wrap major operations and functionality in this class class ParserCommands: @@ -165,13 +165,6 @@ def restart_parser_topologies(self): self.start_parser_topologies() Logger.info('Done restarting the parser topologies') - def init_config(self): - Logger.info('Loading config into ZooKeeper') - Execute(format( - "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), - path=format("{java_home}/bin") - ) - def topologies_exist(self): cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = cmd_open.communicate() @@ -185,27 +178,19 @@ def topologies_exist(self): return True return False - def topologies_running(self): - #TODO - use Storm Rest Commands - # cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # (stdout, stderr) = cmd_open.communicate() - # stdout_lines = stdout.splitlines() - # all_running = False - # if stdout_lines: - # all_running = True - # status_lines = self.__get_status_lines(stdout_lines) - # for parser in self.get_parser_list(): - # parser_found = False - # is_running = False - # for line in status_lines: - # items = re.sub('[\s]+', ' ', line).split() - # if items and items[0] == parser: - # status = items[1] - # parser_found = True - # is_running = self.__is_running(status) - # all_running &= parser_found and is_running - # return all_running - return True + def topologies_running(self, env): + from params import params + env.set_params(params) + all_running = True + topologies = metron_service.get_running_topologies() + for parser in self.get_parser_list(): + if parser in topologies: + parser_found = True + is_running = topologies[parser] in ['ACTIVE','REBALANCING'] + all_running &= parser_found and is_running + else: + all_running = False + return all_running def __get_status_lines(self, lines): status_lines = [] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index c145777d47..4da70e3280 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -22,7 +22,7 @@ from resource_management.libraries.script import Script from resource_management.core.resources.system import Directory, File from parser_commands import ParserCommands - +import metron_service class ParserMaster(Script): def get_component_name(self): @@ -52,7 +52,8 @@ def install(self, env): owner=params.metron_user, content="{}" ) - commands.init_config() + metron_service.init_config() + def configure(self, env, upgrade_type=None, config_dir=None): from params import params @@ -75,10 +76,8 @@ def stop(self, env, upgrade_type=None): def status(self, env): # from params import params # env.set_params(params) - # # commands = ParserCommands(params) - # - # if not commands.topologies_running(): + # if not commands.topologies_running(env): raise ComponentIsNotRunning() def restart(self, env): @@ -87,5 +86,13 @@ def restart(self, env): commands = ParserCommands(params) commands.restart_parser_topologies() + def servicechecktest(self,env): + from params import params + env.set_params(params) + from service_check import ServiceCheck + service_check = ServiceCheck() + Logger.info('Service Check Test') + service_check.service_check(env) + if __name__ == "__main__": ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py index 947c0b50b4..5ab39fc199 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py @@ -20,7 +20,6 @@ from __future__ import print_function from resource_management.libraries.script import Script - from parser_commands import ParserCommands @@ -29,7 +28,7 @@ def service_check(self, env): from params import params env.set_params(params) commands = ParserCommands(params) - all_found = commands.topologies_running() + all_found = commands.topologies_running(env) if all_found: exit(0) else: From 13c50771c61cea3ec0f305d48ba00f5659a93056 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 25 Aug 2016 08:08:09 -0400 Subject: [PATCH 37/82] Initial enrichment commit + mysql setup --- .../configuration/metron-enrichment.xml | 60 +++++++ .../ENRICHMENT/0.2.0BETA/metainfo.xml | 163 ++++++++++++++++++ .../0.2.0BETA/package/files/addMysqlUser.sh | 41 +++++ .../package/files/createMysqlGeoIp.sh | 40 +++++ .../package/files/removeMysqlUser.sh | 33 ++++ .../0.2.0BETA/package/scripts/commands.py | 135 +++++++++++++++ .../package/scripts/enrichment_master.py | 68 ++++++++ .../0.2.0BETA/package/scripts/mysql_server.py | 61 +++++++ .../package/scripts/mysql_service.py | 46 +++++ .../0.2.0BETA/package/scripts/mysql_users.py | 59 +++++++ .../0.2.0BETA/package/scripts/mysql_utils.py | 50 ++++++ .../0.2.0BETA/package/scripts/params.py | 29 ++++ .../0.2.0BETA/package/scripts/params_linux.py | 96 +++++++++++ .../package/scripts/params_windows.py | 18 ++ .../package/scripts/service_check.py | 37 ++++ .../package/scripts/status_params.py | 8 + .../0.2.0BETA/quicklinks/quicklinks.json | 28 +++ .../ENRICHMENT/0.2.0BETA/service_advisor.py | 64 +++++++ .../ENRICHMENT/0.2.0BETA/service_advisor.pyc | Bin 0 -> 3592 bytes .../HDP/2.4/services/ENRICHMENT/metainfo.xml | 26 +++ .../ENRICHMENT/role_command_order.json | 9 + .../docker/rpm-docker/SPECS/metron.spec | 2 + .../src/main/assembly/assembly.xml | 12 ++ .../src/main/resources/ddl/geoip_ddl.sql | 49 ++++++ 24 files changed, 1134 insertions(+) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/addMysqlUser.sh create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/createMysqlGeoIp.sh create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/removeMysqlUser.sh create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_service.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/quicklinks/quicklinks.json create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/metainfo.xml create mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json create mode 100644 metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml new file mode 100755 index 0000000000..1d87adfc70 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml @@ -0,0 +1,60 @@ + + + + + + metron_home + /usr/metron/0.2.0BETA + Metron home directory + Metron home + + + metron_apps_hdfs_dir + /apps/metron + Metron apps HDFS dir + Metron apps HDFS dir + + + metron_user + metron + USER + The user for Metron + Metron User + + + metron_group + metron + The group for Metron + + + metron_enrichment_db_user + root + Database username to use to connect to the database. + + + metron_enrichment_db_password + + PASSWORD + GeoIP Database Password + Password to use against database + + password + false + + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml new file mode 100755 index 0000000000..9c42581909 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml @@ -0,0 +1,163 @@ + + + + 2.0 + + + ENRICHMENT + Enrichment + Enrichment + 0.2.0BETA + + + + ENRICHMENT_MASTER + Enrichment Master + MASTER + 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + KAFKA/KAFKA_BROKER + cluster + + true + + + + + + PYTHON + 600 + + + + + ENRICHMENT_MYSQL_SERVER + MySQL Server + MASTER + 1 + false + + + + PYTHON + + + + CLEAN + + + PYTHON + 600 + + + + + + + + + any + + + metron-common + + + metron-enrichment + + + metron-elasticsearch + + + metron-elasticsearch + + + mysql-connector-java + true + + + + + redhat7 + + + mysql-community-release + true + + + mysql-community-server + true + + MySQL-python + + + + + + + + + + + + + + + + + + + + PYTHON + 300 + + + + HDFS + KAFKA + STORM + ZOOKEEPER + + + + metron-enrichment + + + + + + + + + + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/addMysqlUser.sh new file mode 100755 index 0000000000..c8c2d68636 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/addMysqlUser.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +mysqldbuser=$2 +mysqldbpasswd=$3 +mysqldbhost=$4 +myhostname=$(hostname -f) + +service $mysqldservice start +echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost" +mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';" +mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';" +mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';" +mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';" +if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then + echo "Adding user $mysqldbuser@$myhostname"; + mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';"; + mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$myhostname';"; +fi +mysql -u root -e "flush privileges;" +service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/createMysqlGeoIp.sh new file mode 100755 index 0000000000..cbe14cdb24 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/createMysqlGeoIp.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +geoipscript=$2 + +# Download and extract the actual GeoIP files +mkdir -p /tmp/geoip +mkdir -p /var/lib/mysql-files/ + +pushd /tmp/geoip +curl -O http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz +tar xf GeoLiteCity-latest.tar.xz +cp /tmp/geoip/*/*.csv /var/lib/mysql-files/ +popd + +# Load MySQL with the GeoIP data +service ${mysqldservice} start +mysql -u root < ${geoipscript} +mysql -u root -e "show databases;" +service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/removeMysqlUser.sh new file mode 100755 index 0000000000..33a5ee5b0c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/files/removeMysqlUser.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +mysqldbuser=$2 +userhost=$3 +myhostname=$(hostname -f) +sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E" + +${sudo_prefix} service ${mysqldservice} start +echo "Removing user $mysqldbuser@$userhost" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\"" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\"" +${sudo_prefix} service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py new file mode 100755 index 0000000000..897820d8b6 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import subprocess +import time + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute + + +# Wrap major operations and functionality in this class +class Commands: + __params = None + __enrichment = None + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__enrichment = params.metron_enrichment_topology + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + command_template = """{}/kafka-topics.sh \ + --zookeeper {} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""" + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = 10 + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating topics for enrichment") + + Logger.info("Creating topic'{}'".format(self.__enrichment)) + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + self.__enrichment, + num_partitions, + replication_factor, + retention_bytes)) + Logger.info("Done creating Kafka topics") + + def start_enrichment_topology(self): + Logger.info("Starting Metron enrichment topology: {}".format(self.__enrichment)) + start_cmd_template = """{}/bin/start_enrichment_topology.sh \ + -s {} \ + -z {}""" + Logger.info('Starting ' + self.__enrichment) + Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment, self.__params.zookeeper_quorum)) + + Logger.info('Finished starting enrichment topology') + + def stop_enrichment_topology(self): + Logger.info('Stopping ' + self.__enrichment) + stop_cmd = 'storm kill ' + self.__enrichment + Execute(stop_cmd) + Logger.info('Done stopping enrichment topologies') + + def restart_enrichment_topology(self): + Logger.info('Restarting the enrichment topologies') + self.stop_enrichment_topology() + + # Wait for old topology to be cleaned up by Storm, before starting again. + retries = 0 + topology_active = self.is_topology_active() + while topology_active and retries < 3: + Logger.info('Existing topology still active. Will wait and retry') + time.sleep(40) + topology_active = self.is_topology_active() + retries += 1 + + if not topology_active: + self.start_enrichment_topology() + Logger.info('Done restarting the enrichment topology') + else: + Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + + def is_topology_active(self): + cmd_retrieve = "storm list | grep 'enrichment'" + proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + Logger.info("Retrieval response is: %s" % stdout) + Logger.warning("Error response is: %s" % stderr) + + fields = stdout.split() + if len(fields) < 2: + Logger.warning("Enrichment topology is not running") + return False + + # Get the second column, which is status. We already know first column is enrichment) + status = stdout.split()[1] + running_status_set = {'ACTIVE', 'REBALANCING'} + return status in running_status_set diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py new file mode 100755 index 0000000000..e62bacb4ca --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py @@ -0,0 +1,68 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.libraries.script import Script + +from commands import Commands + + +class Enrichment(Script): + def install(self, env): + import params + env.set_params(params) + commands = Commands(params) + commands.setup_repo() + Logger.info('Install RPM packages') + self.install_packages(env) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + commands.start_enrichment_topology() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.stop_enrichment_topology() + + def status(self, env): + import status_params + env.set_params(status_params) + commands = Commands(status_params) + + if not commands.is_topology_active(): + raise ComponentIsNotRunning() + + def restart(self, env): + import params + env.set_params(params) + commands = Commands(params) + commands.restart_enrichment_topology() + + def kafkabuild(self, env, upgrade_type=None): + import params + env.set_params(params) + commands = Commands(params) + commands.init_kafka_topics() + + +if __name__ == "__main__": + Enrichment().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py new file mode 100755 index 0000000000..2241580fcc --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script + +import mysql_users +from mysql_service import mysql_service +from mysql_utils import mysql_configure + + +class MysqlServer(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def clean(self, env): + import params + env.set_params(params) + mysql_users.mysql_deluser() + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + mysql_configure() + + def start(self, env, rolling_restart=False): + import params + env.set_params(params) + mysql_service(daemon_name=params.daemon_name, action='start') + + def stop(self, env, rolling_restart=False): + import params + env.set_params(params) + mysql_service(daemon_name=params.daemon_name, action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + + mysql_service(daemon_name=status_params.daemon_name, action='status') + + +if __name__ == "__main__": + MysqlServer().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_service.py new file mode 100755 index 0000000000..7c4a7e881a --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_service.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import ComponentIsNotRunning, Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.format import format + + +def mysql_service(daemon_name=None, action='start'): + status_cmd = format("pgrep -l '^{process_name}$'") + cmd = ('service', daemon_name, action) + + if action == 'status': + try: + Execute(status_cmd) + except Fail: + raise ComponentIsNotRunning() + elif action == 'stop': + Execute(cmd, + logoutput=True, + only_if=status_cmd, + sudo=True, + ) + elif action == 'start': + Execute(cmd, + logoutput=True, + not_if=status_cmd, + sudo=True, + ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py new file mode 100755 index 0000000000..186bcb5c1f --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import StaticFile +from resource_management.libraries.functions.format import format + + +# Used to add metron access to the needed components +def mysql_adduser(): + import params + + File(params.mysql_adduser_path, + mode=0755, + content=StaticFile('addMysqlUser.sh') + ) + + # TODO figure out password + add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}") + Execute(add_user_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) + + +# Removes hive metron from components +def mysql_deluser(): + import params + + File(params.mysql_deluser_path, + mode=0755, + content=StaticFile('removeMysqlUser.sh') + ) + + del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}") + Execute(del_user_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py new file mode 100755 index 0000000000..470fe551a7 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import StaticFile +from resource_management.libraries.functions.format import format + +import mysql_users + + +def mysql_configure(): + import params + + # required for running hive + replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname) + Execute(replace_bind_address, + sudo=True, + ) + + # this also will start mysql-server + mysql_users.mysql_adduser() + + File(params.mysql_create_geoip_path, + mode=0755, + content=StaticFile('createMysqlGeoIp.sh') + ) + + geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geo_ip_ddl}") + Execute(geoip_setup_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py new file mode 100755 index 0000000000..6440005faf --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.expect import expect + +if OSCheck.is_windows_family(): + pass +else: + from params_linux import * + +java_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py new file mode 100755 index 0000000000..a6c6e09d97 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os + +from ambari_commons.os_check import OSCheck +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.script import Script +from resource_management.libraries.functions.is_empty import is_empty + +import status_params + +# Server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +hostname = config['hostname'] +metron_home = config['configurations']['metron-enrichment']['metron_home'] +metron_ddl_dir = metron_home + '/ddl' +geo_ip_ddl = metron_ddl_dir + '/geoip_ddl.sql' +metron_enrichment_topology = status_params.metron_enrichment_topology +yum_repo_type = 'local' + +# Hadoop params +hadoop_home_dir = stack_select.get_hadoop_dir("home") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + +# Zookeeper +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +has_zk_host = not len(zk_hosts) == 0 +zookeeper_quorum = None +if has_zk_host: + if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: + zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] + else: + zookeeper_clientPort = '2181' + zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) + # last port config + zookeeper_quorum += ':' + zookeeper_clientPort + +# Kafka +stack_root = Script.get_stack_root() +kafka_home = os.path.join(stack_root, "current", "kafka-broker") +kafka_bin_dir = os.path.join(kafka_home, "bin") +metron_enrichment_topic_retention = config['configurations']['metron-enrichment']['metron_enrichment_topic_retention'] + +kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) +has_kafka_host = not len(kafka_hosts) == 0 +kafka_brokers = None +if has_kafka_host: + if 'port' in config['configurations']['kafka-broker']: + kafka_broker_port = config['configurations']['kafka-broker']['port'] + else: + kafka_broker_port = '6667' + kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts']) + kafka_brokers += ':' + kafka_broker_port + +# MYSQL +if OSCheck.is_ubuntu_family(): + mysql_configname = '/etc/mysql/my.cnf' +else: + mysql_configname = '/etc/my.cnf' + +daemon_name = status_params.daemon_name +mysql_user = "mysql" +mysql_group = 'mysql' +mysql_host = config['clusterHostInfo']['enrichment_mysql_host'] + +mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" +mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" +mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh" + +enrichment_hosts = default("/clusterHostInfo/enrichment_host", []) +enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None + +metron_user = config['configurations']['metron-enrichment']['metron_user'] +enrichment_metron_user_passwd = config['configurations']['metron-enrichment']['metron_enrichment_db_password'] +enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(enrichment_metron_user_passwd) else enrichment_metron_user_passwd +process_name = status_params.process_name diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py new file mode 100755 index 0000000000..12374f76f7 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py @@ -0,0 +1,18 @@ + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +raise NotImplementedError diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py new file mode 100755 index 0000000000..c02d695919 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import print_function + +from resource_management.libraries.script import Script + +from commands import Commands + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + commands = Commands(params) + if commands.is_topology_active(): + exit(0) + else: + exit(1) + + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py new file mode 100755 index 0000000000..cf56b19ab0 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py @@ -0,0 +1,8 @@ +from ambari_commons import OSCheck + +metron_enrichment_topology = 'enrichment' +process_name = 'mysqld' +if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): + daemon_name = 'mysql' +else: + daemon_name = 'mysqld' \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/quicklinks/quicklinks.json new file mode 100755 index 0000000000..ee1b225084 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py new file mode 100755 index 0000000000..ce4646eb7c --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py @@ -0,0 +1,64 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE)) + + +class ENRICHMENT020BETAServiceAdvisor(service_advisor.ServiceAdvisor): + # colocate Metron Parser Master with KAFKA_BROKERs + def colocateService(self, hostsComponentsMap, serviceComponents): + enrichmentMasterComponent = [component for component in serviceComponents if + component["StackServiceComponents"]["component_name"] == "ENRICHMENT_MASTER"] + enrichmentMasterComponent = enrichmentMasterComponent[0] + if not self.isComponentHostsPopulated(enrichmentMasterComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "ENRICHMENT_MASTER"} not in hostComponents: + hostsComponentsMap[hostName].append({"name": "ENRICHMENT_MASTER"}) + if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "ENRICHMENT_MASTER"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "ENRICHMENT_MASTER"}) + + def getServiceComponentLayoutValidations(self, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + + enrichmentHosts = self.getHosts(componentsList, "ENRICHMENT_MASTER") + brokerHosts = self.getHosts(componentsList, "KAFKA_BROKER") + + items = [] + + mismatchHosts = sorted(set(enrichmentHosts).symmetric_difference(set(brokerHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Metron Enrichment Master must be installed on Kafka Brokers. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString) + items.append( + {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'ENRICHMENT_MASTER'}) + + return items diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc new file mode 100755 index 0000000000000000000000000000000000000000..b8d43a3702be7f169e4874c340e110f7f45e172b GIT binary patch literal 3592 zcmcImUvC@75uYO^N;*llBgYBUqV3u(61tH|#SeZ+iz1Xn)r2IhEwW^h`uFJ_krk{`|6>*arT+Llw&+y=jqXwZ0d--P zO@Aildwu>1+v}Fu73%u}k$oKcGFCDWIuZIpirUcg@#iL8J@!&54wF$FcsfjCQLed% zj|E<3Dw0^*Nh$^jSo|c`Y1kQQ5TZ<+@Or6~0~u>o5yF+y$Znpt8vAw84I^m>p~^Pl z=y9kozz`}iPSQ)!g&Z#kLO#@sL>PlGFe=!o?0IPr#yuQ5oTOpzLW^V^%T!&2L!8uN z3*5uG8Y+`&4upM^WR$Ca%hKEdJK`HiVKbjrc10Ns_M?31(c|Bltpjf&;zWy)VsoG^ z%N6pCFNYdhLZ!hl3cc8uc5aarxd^EC8i%ulq@z7Bd&UBhbd6_nHsNVo;B%q%@T+Qd zJRVm(6OT%g_Nr0lTU9-V7n^SV2{g4|#*tJ?r1EMMrtoEFBD^8g@VyR{i@dR5Z*nwD zh;i~bg=gd5j!=1w--|^SUiQyiCm4D=Sn!Y+i$^tAG~7pGujV$~9s8`&dVcz{CC+M& zQ){*wbyu7^V*j*x&}cPIn|M1EwdQMa)My^;2#I(DL-NfqW!<4*$exoy#hycDemG-+ zl~F_KhuzSJMR9NB^`z(}*D{S63PYI=LdCIDP%W^da1d&P7j-M#xx|Vc{(w(Y-9rjg zD%I?#wC7){s@mebkm)r%-S&d(P$g+)IMGNHN;~@QPghlu#SAB^ivL3ofG}>35-$)# zj`TXtdbNwsqsW+X*{_~T1v$gwlcdy>6x!Lix>~tS2=3`;p!=rwNL85pe zt%=h*t5!TIlH-c+o)QX1xi<#_FaLl zR%o_F(a0*|TYhE=!dD z!;o-RPFi77&fhQGT7+;brXL9aw|k2xm+3A96mKoEfGd~qJA)j!#x1i4n)(xrE{8XF z>^`y-)eDtL-@y_mt$A(7-azV)K)MM@`=sW!>P`qE=?x4=wZo%Yd(SyNsyliK1m0BU ziQdK#s)h9DiPGvtG8{#omVx2DloO>(U_*k+I55*x4uE5$s^L*G>S{*e1@B)-Qzzcg z=$3=&W9+h8% zq&~(#)}|#a+q!Gr$M~7`b36vmtnM-%gXbSGVLStGK=Hx1_vV-eo*BdIjA3S20!V*h zK)Q?>*h&N#0rK1573u0JU6pAD9+|zM+$cVEI;#OU{VViF7Wni{RQ=9rL6e91|xM4pFa2wb_ zB5hKzi8s*i&@)Ck;0<24%wBAokmfE{+6HNX1?3d+>afooMx8;i;`)Hn{}=q7NS&gZ z*B5XR8D7L-q_pTr^fc7Ph)WvSj=b)rC-&0h5`D5_I&C+JqGZe^`0egYFqX>du?P~= zB^4^L>P}4c&V$X}HI=GrlW~iwN6u~9LZwFzwD^aX{X&54;4(D_~ z31hQ4yMQ?C-ZCrea4m3HhEdDps!=PCgvsEATIv{g9ey&FQ77U#9}sFec)to6Uejo9 zvN$t>uWTLT{p_9n!TXio#0(7(S0w-9hSwhjDG*;|8}?+dp!drWqSAY7Q + + + + 2.0 + + + ENRICHMENT + 0.2.0BETA + common-services/ENRICHMENT/0.2.0BETA + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json new file mode 100755 index 0000000000..6fcd42abb8 --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json @@ -0,0 +1,9 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START"], + "ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["ENRICHMENT_MASTER-START"] + } +} diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 18f810ff39..d53843ec43 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -220,6 +220,7 @@ This package installs the Metron Enrichment files %dir %{metron_home}/config %dir %{metron_home}/config/zookeeper %dir %{metron_home}/config/zookeeper/enrichments +%dir %{metron_home}/ddl %dir %{metron_home}/flux %dir %{metron_home}/flux/enrichment %{metron_home}/bin/latency_summarizer.sh @@ -229,6 +230,7 @@ This package installs the Metron Enrichment files %{metron_home}/config/zookeeper/enrichments/snort.json %{metron_home}/config/zookeeper/enrichments/websphere.json %{metron_home}/config/zookeeper/enrichments/yaf.json +%{metron_home}/ddl/geoip_ddl.sql %{metron_home}/flux/enrichment/remote.yaml %exclude %{metron_home}/flux/enrichment/test.yaml %attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml index 4358370e29..7f9c013ab3 100644 --- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml +++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml @@ -42,6 +42,18 @@ unix true + + ${project.basedir}/src/main/resources/ddl + /ddl + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + ${project.basedir}/src/main/flux /flux diff --git a/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql new file mode 100644 index 0000000000..02616c6dfc --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql @@ -0,0 +1,49 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +CREATE DATABASE IF NOT EXISTS GEO; + +USE GEO; + +DROP TABLE IF EXISTS `blocks`; +CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID` +int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) ) +ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1; + +DROP TABLE IF EXISTS `location`; +CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2) + default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default +NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY + (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED; + +load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; +load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; + + +DELIMITER $$ +DROP FUNCTION IF EXISTS `IPTOLOCID` $$ +CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned + BEGIN + DECLARE ipn INTEGER UNSIGNED; + DECLARE locID_var INTEGER; + IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0; + END IF; + SET ipn = INET_ATON(ip); + SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn; + RETURN locID_var; + END +$$ +DELIMITER ; From b0b3e7f826fdf8f50ac27957c6380c7dad7032d9 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 30 Aug 2016 11:27:36 -0400 Subject: [PATCH 38/82] Adding restart and status --- .../ENRICHMENT/0.2.0BETA/metainfo.xml | 26 +++++++++--------- .../0.2.0BETA/package/scripts/commands.py | 14 +++++++++- .../package/scripts/enrichment_master.py | 6 +++- .../0.2.0BETA/package/scripts/params_linux.py | 3 ++ .../package/scripts/status_params.py | 2 ++ .../ENRICHMENT/0.2.0BETA/service_advisor.py | 2 +- .../ENRICHMENT/0.2.0BETA/service_advisor.pyc | Bin 3592 -> 0 bytes .../ENRICHMENT/role_command_order.json | 1 + 8 files changed, 38 insertions(+), 16 deletions(-) delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml index 9c42581909..afc80da480 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml @@ -119,19 +119,19 @@ MySQL-python - - - - - - - - - - - - - + + redhat6 + + + mysql-server + true + + + mysql + true + + + diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py index 897820d8b6..86803b3794 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py @@ -15,23 +15,35 @@ limitations under the License. """ +import os import subprocess import time from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Execute, File # Wrap major operations and functionality in this class class Commands: __params = None __enrichment = None + __configured = False def __init__(self, params): if params is None: raise ValueError("params argument is required for initialization") self.__params = params self.__enrichment = params.metron_enrichment_topology + self.__configured = os.path.isfile(self.__params.configured_flag_file) + + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) def setup_repo(self): def local_repo(): diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py index e62bacb4ca..9b24e061b3 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py @@ -34,7 +34,11 @@ def start(self, env, upgrade_type=None): import params env.set_params(params) commands = Commands(params) - commands.init_kafka_topics() + + if not commands.is_configured(): + commands.init_kafka_topics() + commands.set_configured() + commands.start_enrichment_topology() def stop(self, env, upgrade_type=None): diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py index a6c6e09d97..3e132a0774 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py @@ -37,6 +37,9 @@ metron_enrichment_topology = status_params.metron_enrichment_topology yum_repo_type = 'local' +metron_config_path = metron_home + '/config' +configured_flag_file = metron_config_path + '/metron_enrichment_is_configured' + # Hadoop params hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py index cf56b19ab0..d7eb0d125b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py @@ -1,6 +1,8 @@ from ambari_commons import OSCheck metron_enrichment_topology = 'enrichment' +configured_flag_file = "" + process_name = 'mysqld' if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): daemon_name = 'mysql' diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py index ce4646eb7c..7399a82e1b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.py @@ -31,7 +31,7 @@ class ENRICHMENT020BETAServiceAdvisor(service_advisor.ServiceAdvisor): - # colocate Metron Parser Master with KAFKA_BROKERs + # colocate Metron Enrichment Master with KAFKA_BROKERs def colocateService(self, hostsComponentsMap, serviceComponents): enrichmentMasterComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "ENRICHMENT_MASTER"] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/service_advisor.pyc deleted file mode 100755 index b8d43a3702be7f169e4874c340e110f7f45e172b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3592 zcmcImUvC@75uYO^N;*llBgYBUqV3u(61tH|#SeZ+iz1Xn)r2IhEwW^h`uFJ_krk{`|6>*arT+Llw&+y=jqXwZ0d--P zO@Aildwu>1+v}Fu73%u}k$oKcGFCDWIuZIpirUcg@#iL8J@!&54wF$FcsfjCQLed% zj|E<3Dw0^*Nh$^jSo|c`Y1kQQ5TZ<+@Or6~0~u>o5yF+y$Znpt8vAw84I^m>p~^Pl z=y9kozz`}iPSQ)!g&Z#kLO#@sL>PlGFe=!o?0IPr#yuQ5oTOpzLW^V^%T!&2L!8uN z3*5uG8Y+`&4upM^WR$Ca%hKEdJK`HiVKbjrc10Ns_M?31(c|Bltpjf&;zWy)VsoG^ z%N6pCFNYdhLZ!hl3cc8uc5aarxd^EC8i%ulq@z7Bd&UBhbd6_nHsNVo;B%q%@T+Qd zJRVm(6OT%g_Nr0lTU9-V7n^SV2{g4|#*tJ?r1EMMrtoEFBD^8g@VyR{i@dR5Z*nwD zh;i~bg=gd5j!=1w--|^SUiQyiCm4D=Sn!Y+i$^tAG~7pGujV$~9s8`&dVcz{CC+M& zQ){*wbyu7^V*j*x&}cPIn|M1EwdQMa)My^;2#I(DL-NfqW!<4*$exoy#hycDemG-+ zl~F_KhuzSJMR9NB^`z(}*D{S63PYI=LdCIDP%W^da1d&P7j-M#xx|Vc{(w(Y-9rjg zD%I?#wC7){s@mebkm)r%-S&d(P$g+)IMGNHN;~@QPghlu#SAB^ivL3ofG}>35-$)# zj`TXtdbNwsqsW+X*{_~T1v$gwlcdy>6x!Lix>~tS2=3`;p!=rwNL85pe zt%=h*t5!TIlH-c+o)QX1xi<#_FaLl zR%o_F(a0*|TYhE=!dD z!;o-RPFi77&fhQGT7+;brXL9aw|k2xm+3A96mKoEfGd~qJA)j!#x1i4n)(xrE{8XF z>^`y-)eDtL-@y_mt$A(7-azV)K)MM@`=sW!>P`qE=?x4=wZo%Yd(SyNsyliK1m0BU ziQdK#s)h9DiPGvtG8{#omVx2DloO>(U_*k+I55*x4uE5$s^L*G>S{*e1@B)-Qzzcg z=$3=&W9+h8% zq&~(#)}|#a+q!Gr$M~7`b36vmtnM-%gXbSGVLStGK=Hx1_vV-eo*BdIjA3S20!V*h zK)Q?>*h&N#0rK1573u0JU6pAD9+|zM+$cVEI;#OU{VViF7Wni{RQ=9rL6e91|xM4pFa2wb_ zB5hKzi8s*i&@)Ck;0<24%wBAokmfE{+6HNX1?3d+>afooMx8;i;`)Hn{}=q7NS&gZ z*B5XR8D7L-q_pTr^fc7Ph)WvSj=b)rC-&0h5`D5_I&C+JqGZe^`0egYFqX>du?P~= zB^4^L>P}4c&V$X}HI=GrlW~iwN6u~9LZwFzwD^aX{X&54;4(D_~ z31hQ4yMQ?C-ZCrea4m3HhEdDps!=PCgvsEATIv{g9ey&FQ77U#9}sFec)to6Uejo9 zvN$t>uWTLT{p_9n!TXio#0(7(S0w-9hSwhjDG*;|8}?+dp!drWqSAY7Q Date: Tue, 30 Aug 2016 13:03:18 -0400 Subject: [PATCH 39/82] Now with working status and service checks. --- .../0.2.0BETA/configuration/metron-env.xml | 20 +++++++- .../METRON/0.2.0BETA/metainfo.xml | 2 +- .../package/scripts/indexing_commands.py | 46 +++++++++++-------- .../package/scripts/indexing_master.py | 32 ++++--------- .../package/scripts/metron_service.py | 31 ++++++++++++- .../package/scripts/params/params_linux.py | 14 ++---- .../package/scripts/parser_commands.py | 17 +++---- .../package/scripts/parser_master.py | 36 +++++---------- .../package/scripts/service_check.py | 7 +-- .../package/scripts/status_params.py | 39 ++++++++++++++++ 10 files changed, 153 insertions(+), 91 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index 4bda11c6b3..6d08e630bd 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -76,7 +76,15 @@ Comma delimited list of Elasticsearch URLs. (eshost1:9200,eshost2:9200) - content + storm_rest_addr + + true + false + + + + + global-json global.json template This is the jinja template for global.json file @@ -90,4 +98,14 @@ content + + elasticsearch-properties + elasticsearch.properties template + This is the jinja template for elasticsearch.properties file + + + + elasticsearch-properties + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml index c57b381476..827cfef0e5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -21,7 +21,7 @@ METRON Metron - A scalable advanced security analytics framework built with on Hadoop + A scalable advanced security analytics framework built on Hadoop 0.2.0BETA diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 2e0d5cef29..5fcdc51151 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -22,16 +22,31 @@ from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions import format +import metron_service +import os + + # Wrap major operations and functionality in this class class IndexingCommands: __params = None __indexing = None + __configured = False def __init__(self, params): if params is None: raise ValueError("params argument is required for initialization") self.__params = params self.__indexing = params.metron_indexing_topology + self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file) + + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.parsers_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) def setup_repo(self): def local_repo(): @@ -126,22 +141,15 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) - - def is_topology_active(self): - # cmd_retrieve = "storm list | grep 'indexing'" - # proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - # (stdout, stderr) = proc.communicate() - # Logger.info("Retrieval response is: %s" % stdout) - # Logger.warning("Error response is: %s" % stderr) - # - # fields = stdout.split() - # if len(fields) < 2: - # Logger.warning("Indexing topology is not running") - # return False - # - # # Get the second column, which is status. We already know first column is indexing) - # status = stdout.split()[1] - # running_status_set = {'ACTIVE', 'REBALANCING'} - # return status in running_status_set - # - return True + def is_topology_active(self,env): + env.set_params(self.__params) + active = True + topologies = metron_service.get_running_topologies() + is_running = False + if 'indexing' in topologies: + is_running = topologies['indexing'] in ['ACTIVE','REBALANCING'] + active &= is_running + if active == False: + raise ValueError(str(topologies)) + + return active diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index 899d9d9fe5..5a34c4f2e9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -36,33 +36,21 @@ def install(self, env): self.install_packages(env) def configure(self, env): - from params import params env.set_params(params) - Logger.info("Configure Metron global.json") - - directories = [params.metron_zookeeper_config_path] - Directory(directories, - # recursive=True, - mode=0755, - owner=params.metron_user, - group=params.metron_group - ) - - File("{}/global.json".format(params.metron_zookeeper_config_path), - owner=params.metron_user, - content=InlineTemplate(params.global_json_template) - ) commands = IndexingCommands(params) metron_service.init_config() + if not commands.is_configured(): + commands.init_kafka_topics() + commands.set_configured() + def start(self, env, upgrade_type=None): from params import params env.set_params(params) self.configure(env) commands = IndexingCommands(params) - commands.init_kafka_topics() commands.start_indexing_topology() def stop(self, env, upgrade_type=None): @@ -72,13 +60,11 @@ def stop(self, env, upgrade_type=None): commands.stop_indexing_topology() def status(self, env): - # from params import params - # env.set_params(params) - # - # commands = IndexingCommands(params) - # - # if not commands.is_topology_active(): - raise ComponentIsNotRunning() + import status_params + env.set_params(status_params) + commands = IndexingCommands(status_params) + if not commands.is_topology_active(env): + raise ComponentIsNotRunning() def restart(self, env): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py index 2f39848d8d..5f5fa06c1c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -17,11 +17,13 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions import format -from resource_management.libraries.script import Script +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import InlineTemplate import subprocess import json + def init_config(): Logger.info('Loading config into ZooKeeper') Execute(format( @@ -29,6 +31,7 @@ def init_config(): path=format("{java_home}/bin") ) + def get_running_topologies(): Logger.info('Getting Running Storm Topologies from Storm REST Server') @@ -36,10 +39,34 @@ def get_running_topologies(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() - stormjson = json.loads(stdout) + try: + stormjson = json.loads(stdout) + except ValueError: + return {} + topologiesDict = {} for topology in stormjson['topologies']: topologiesDict[topology['name']] = topology['status'] + Logger.info("Topologies: " + str(topologiesDict)) return topologiesDict + + +def load_global_config(params): + Logger.info('Create Metron Local Config Directory') + Logger.info("Configure Metron global.json") + + directories = [params.metron_zookeeper_config_path] + Directory(directories, + # recursive=True, + mode=0755, + owner=params.metron_user, + group=params.metron_group + ) + + File("{}/global.json".format(params.metron_zookeeper_config_path), + owner=params.metron_user, + content=InlineTemplate(params.global_json_template) + ) + init_config() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 0904ddc250..77c3c06898 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -40,8 +40,9 @@ metron_group = config['configurations']['metron-env']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured' -global_json_template = config['configurations']['metron-env']['content'] +parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' +global_json_template = config['configurations']['metron-env']['global-json'] es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] es_url = config['configurations']['metron-env']['es_url'] yum_repo_type = 'local' @@ -68,14 +69,7 @@ zookeeper_quorum += ':' + zookeeper_clientPort #Storm -storm_ui_hosts = default("/clusterHostInfo/storm_ui_server_hosts", []) -has_storm_host = not len(storm_ui_hosts) == 0 -if has_storm_host: - if 'storm-site' in config['configurations'] and 'ui.port' in config['configurations']['storm-site']: - storm_ui_port = config['configurations']['storm-site']['ui.port'] - else: - storm_ui_port = '8744' - storm_rest_addr = storm_ui_hosts[0] + ':' + storm_ui_port +storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] # Kafka kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index bf1916e32d..3803bbee5c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -40,7 +40,7 @@ def __init__(self, params): raise ValueError("params argument is required for initialization") self.__params = params self.__parser_list = self.__get_parsers(params) - self.__configured = os.path.isfile(self.__params.configured_flag_file) + self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file) # get list of parsers def __get_parsers(self, params): @@ -50,7 +50,7 @@ def is_configured(self): return self.__configured def set_configured(self): - File(self.__params.configured_flag_file, + File(self.__params.parsers_configured_flag_file, content="", owner=self.__params.metron_user, mode=0775) @@ -178,18 +178,19 @@ def topologies_exist(self): return True return False - def topologies_running(self, env): - from params import params - env.set_params(params) + def topologies_running(self,env): + env.set_params(self.__params) all_running = True topologies = metron_service.get_running_topologies() for parser in self.get_parser_list(): + parser_found = False if parser in topologies: parser_found = True is_running = topologies[parser] in ['ACTIVE','REBALANCING'] - all_running &= parser_found and is_running - else: - all_running = False + all_running &= parser_found and is_running + if all_running == False: + raise ValueError(str(topologies)) + return all_running def __get_status_lines(self, lines): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index 4da70e3280..7bba256bf8 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -20,7 +20,6 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script -from resource_management.core.resources.system import Directory, File from parser_commands import ParserCommands import metron_service @@ -38,33 +37,22 @@ def install(self, env): Logger.info('Install RPM packages') self.install_packages(env) - Logger.info('Create Metron Local Config Directory') - directories = [params.metron_zookeeper_config_path] - Directory(directories, - # recursive=True, - mode=0755, - owner=params.metron_user, - group=params.metron_group - ) - - #Default global.json to empty file - File("{}/global.json".format(params.metron_zookeeper_config_path), - owner=params.metron_user, - content="{}" - ) - metron_service.init_config() - def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) + metron_service.load_global_config(params) + commands = ParserCommands(params) + if not commands.is_configured(): + commands.init_parsers() + commands.init_kafka_topics() + commands.set_configured() def start(self, env, upgrade_type=None): from params import params env.set_params(params) + self.configure(env) commands = ParserCommands(params) - commands.init_parsers() - commands.init_kafka_topics() commands.start_parser_topologies() def stop(self, env, upgrade_type=None): @@ -74,11 +62,11 @@ def stop(self, env, upgrade_type=None): commands.stop_parser_topologies() def status(self, env): - # from params import params - # env.set_params(params) - # commands = ParserCommands(params) - # if not commands.topologies_running(env): - raise ComponentIsNotRunning() + import status_params + env.set_params(status_params) + commands = ParserCommands(status_params) + if not commands.topologies_running(env): + raise ComponentIsNotRunning() def restart(self, env): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py index 5ab39fc199..1af1692f92 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py @@ -21,14 +21,15 @@ from resource_management.libraries.script import Script from parser_commands import ParserCommands +from indexing_commands import IndexingCommands class ServiceCheck(Script): def service_check(self, env): from params import params - env.set_params(params) - commands = ParserCommands(params) - all_found = commands.topologies_running(env) + parsercommands = ParserCommands(params) + indexingcommands = IndexingCommands(params) + all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env) if all_found: exit(0) else: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py new file mode 100644 index 0000000000..79d5a5aa74 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.script import Script + + +config = Script.get_config() +#Parsers +parsers = config['configurations']['metron-env']['parsers'] +metron_home = config['configurations']['metron-env']['metron_home'] +metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') +parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' + +#Indexing +metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' + +#Storm +storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] From 445f9181f918ccc53218ab8a0b4fa9368440e3e2 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 30 Aug 2016 13:57:46 -0400 Subject: [PATCH 40/82] Quicklinks --- .../ENRICHMENT/0.2.0BETA/metainfo.xml | 13 ++++++------- .../0.2.0BETA/package/scripts/status_params.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml index afc80da480..3d01d76a8f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml @@ -151,13 +151,12 @@ metron-enrichment - - - - - - - + + + quicklinks.json + true + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py index d7eb0d125b..8574b85e69 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py @@ -1,3 +1,19 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from ambari_commons import OSCheck metron_enrichment_topology = 'enrichment' From 1a26c772f24df0c6dc25bb7144f736c641127c28 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 30 Aug 2016 15:11:03 -0400 Subject: [PATCH 41/82] Working elasticsearch.properties in config. --- .../0.2.0BETA/configuration/metron-env.xml | 44 +++++++++++++++++-- .../package/scripts/indexing_commands.py | 17 ++++--- .../package/scripts/indexing_master.py | 7 +-- .../package/scripts/metron_service.py | 5 +++ .../package/scripts/params/params_linux.py | 3 +- .../package/scripts/parser_commands.py | 7 +-- .../package/scripts/parser_master.py | 3 +- .../package/scripts/status_params.py | 2 +- 8 files changed, 62 insertions(+), 26 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index 6d08e630bd..3c68bc9f16 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -73,7 +73,7 @@ es_url - Comma delimited list of Elasticsearch URLs. (eshost1:9200,eshost2:9200) + Comma delimited list of Elasticsearch URLs. (eshost1:9300,eshost2:9300) storm_rest_addr @@ -90,7 +90,7 @@ { "es.clustername": "{{ es_cluster_name }}", -"es.ip": "[{{ es_url }}]", +"es.ip": "{{ es_url }}", "es.date.format": "yyyy.MM.dd.HH" } @@ -100,12 +100,48 @@ elasticsearch-properties + The template for the elasticsearch.properties file. elasticsearch.properties template - This is the jinja template for elasticsearch.properties file +##### Storm ##### +indexing.workers=1 +indexing.executors=0 +##### Kafka ##### +kafka.zk={{ zookeeper_quorum }} +kafka.broker={{ kafka_brokers }} +kafka.start=WHERE_I_LEFT_OFF +##### Indexing ##### +index.input.topic=indexing +index.error.topic=indexing_error +writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter +##### Metrics ##### +#reporters +org.apache.metron.metrics.reporter.graphite=true +org.apache.metron.metrics.reporter.console=false +org.apache.metron.metrics.reporter.jmx=false +#Graphite Addresses +org.apache.metron.metrics.graphite.address=localhost +org.apache.metron.metrics.graphite.port=2023 +#TelemetryParserBolt +org.apache.metron.metrics.TelemetryParserBolt.acks=true +org.apache.metron.metrics.TelemetryParserBolt.emits=true +org.apache.metron.metrics.TelemetryParserBolt.fails=true +##### HDFS ##### +bolt.hdfs.batch.size=5000 +bolt.hdfs.field.delimiter=| +bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy +bolt.hdfs.rotation.policy.units=DAYS +bolt.hdfs.rotation.policy.count=1 +bolt.hdfs.file.rotation.size.in.mb=5 +bolt.hdfs.file.system.url={{ default_fs }} +bolt.hdfs.wip.file.path=/paloalto/wip +bolt.hdfs.finished.file.path=/paloalto/rotated +bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec +index.hdfs.output=/tmp/metron/enriched - elasticsearch-properties + content + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 5fcdc51151..3817564309 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -113,30 +113,32 @@ def stop_indexing_topology(self): Execute(stop_cmd) Logger.info('Done stopping indexing topologies') - def restart_indexing_topology(self): + def restart_indexing_topology(self,env): Logger.info('Restarting the indexing topologies') self.stop_indexing_topology() # Wait for old topology to be cleaned up by Storm, before starting again. retries = 0 - topology_active = self.is_topology_active() - while topology_active and retries < 3: + topology_active = self.is_topology_active(env) + while self.is_topology_active(env) and retries < 3: Logger.info('Existing topology still active. Will wait and retry') - time.sleep(40) - topology_active = self.is_topology_active() + time.sleep(10) retries += 1 if not topology_active: + Logger.info('Waiting for storm kill to complete') + time.sleep(30) self.start_indexing_topology() Logger.info('Done restarting the indexing topologies') else: Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + def is_configured(self): return self.__configured def set_configured(self): - File(self.__params.configured_flag_file, + File(self.__params.indexing_configured_flag_file, content="", owner=self.__params.metron_user, mode=0775) @@ -149,7 +151,4 @@ def is_topology_active(self,env): if 'indexing' in topologies: is_running = topologies['indexing'] in ['ACTIVE','REBALANCING'] active &= is_running - if active == False: - raise ValueError(str(topologies)) - return active diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index 5a34c4f2e9..cc4b452fd0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -17,9 +17,6 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate from indexing_commands import IndexingCommands import metron_service @@ -40,7 +37,7 @@ def configure(self, env): env.set_params(params) commands = IndexingCommands(params) - metron_service.init_config() + metron_service.load_global_config(params) if not commands.is_configured(): commands.init_kafka_topics() @@ -71,7 +68,7 @@ def restart(self, env): env.set_params(params) self.configure(env) commands = IndexingCommands(params) - commands.restart_indexing_topology() + commands.restart_indexing_topology(env) if __name__ == "__main__": diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py index 5f5fa06c1c..5fcc9c7540 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -69,4 +69,9 @@ def load_global_config(params): owner=params.metron_user, content=InlineTemplate(params.global_json_template) ) + + File("{}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'), + owner=params.metron_user, + content=InlineTemplate(params.global_properties_template)) + init_config() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 77c3c06898..c6a161a83b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -41,8 +41,9 @@ metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' -indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' global_json_template = config['configurations']['metron-env']['global-json'] +global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] es_url = config['configurations']['metron-env']['es_url'] yum_repo_type = 'local' diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index 3803bbee5c..34b569dcf6 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -153,11 +153,11 @@ def stop_parser_topologies(self): Execute(stop_cmd) Logger.info('Done stopping parser topologies') - def restart_parser_topologies(self): + def restart_parser_topologies(self,env): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() attempt_count = 0 - while self.topologies_exist(): + while self.topologies_running(env): if attempt_count > 2: raise Exception("Unable to kill topologies") attempt_count += 1 @@ -188,9 +188,6 @@ def topologies_running(self,env): parser_found = True is_running = topologies[parser] in ['ACTIVE','REBALANCING'] all_running &= parser_found and is_running - if all_running == False: - raise ValueError(str(topologies)) - return all_running def __get_status_lines(self, lines): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index 7bba256bf8..5bb3d7fdc0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -71,8 +71,9 @@ def status(self, env): def restart(self, env): from params import params env.set_params(params) + self.configure(env) commands = ParserCommands(params) - commands.restart_parser_topologies() + commands.restart_parser_topologies(env) def servicechecktest(self,env): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py index 79d5a5aa74..d14fc24237 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py @@ -33,7 +33,7 @@ #Indexing metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] -indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' #Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] From 1f0db69d63669d5d258c48c357f62b37fe986e07 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 30 Aug 2016 16:31:40 -0400 Subject: [PATCH 42/82] You may find this helpful - adding the pom.xml that I've had locally. --- .../packaging/docker/rpm-docker/pom.xml | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 metron-deployment/packaging/docker/rpm-docker/pom.xml diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml new file mode 100644 index 0000000000..8a4007b5e1 --- /dev/null +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -0,0 +1,212 @@ + + + + + 4.0.0 + metron-rpm + pom + metron-rpm + 0.2.0BETA + + org.apache.metron + metron-deployment + 0.2.0BETA + ../../.. + + RPM Builder for Apache Metron + + ${parent.relativePath}/.. + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.1 + + + validate + + run + + + + ******** Displaying value of property ******** + ${metron_dir} + + + + + + + maven-clean-plugin + 3.0.0 + + + + BUILD + false + + + BUILDROOT + false + + + RPMS + false + + + RPMS/noarch + false + + + SOURCES + false + + + SRPMS + false + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + docker-build + package + + exec + + + docker + + build + -f + Dockerfile + -t + ${rpm.docker.tag} + . + + + + + rpm-build + package + + exec + + + docker + + run + -v + ${project.basedir}:/root + ${rpm.docker.tag}:latest + /bin/bash + -c + ./build.sh ${project.version} + + + + + + + maven-resources-plugin + 3.0.1 + + + copy-rpm-sources + prepare-package + + copy-resources + + + ${project.basedir}/SOURCES + + + ${metron_dir}/metron-platform/metron-common/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-data-management/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-elasticsearch/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-enrichment/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-indexing/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-parsers/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-pcap-backend/target/ + + *.tar.gz + + + + ${metron_dir}/metron-platform/metron-solr/target/ + + *.tar.gz + + + + + + + copy-rpms-target + package + + copy-resources + + + ${project.basedir}/target/RPMS + + + RPMS + + + + + + + + + From 008b547d2ecb8730271a3e18b29d42b07876d8f6 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 30 Aug 2016 17:10:13 -0400 Subject: [PATCH 43/82] tweaks --- .../packaging/docker/rpm-docker/pom.xml | 195 ++++++++++++++++++ .../src/main/assembly/assembly.xml | 2 +- pom.xml | 3 +- 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 metron-deployment/packaging/docker/rpm-docker/pom.xml diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml new file mode 100644 index 0000000000..9d79e46101 --- /dev/null +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -0,0 +1,195 @@ + + + + + 4.0.0 + metron-rpm-docker + pom + metron-rpm-docker + + org.apache.metron + metron-deployment + 0.2.0BETA + ../../.. + + Building and deploying Metron + + UTF-8 + UTF-8 + rpm-docker + + + + + + maven-clean-plugin + 3.0.0 + + + + BUILD + false + + + BUILDROOT + false + + + RPMS + false + + + RPMS/noarch + false + + + SOURCES + false + + + SRPMS + false + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + docker-build + package + + exec + + + docker + + build + -f + Dockerfile + -t + ${rpm.docker.tag} + . + + + + + rpm-build + package + + exec + + + docker + + run + -v + ${project.basedir}:/root + ${rpm.docker.tag}:latest + /bin/bash + -c + ./build.sh ${project.version} + + + + + + + maven-resources-plugin + 3.0.1 + + + copy-rpm-sources + prepare-package + + copy-resources + + + ${project.basedir}/SOURCES + + + ${project.basedir}/../../../../metron-platform/metron-common/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-data-management/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-elasticsearch/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-enrichment/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-indexing/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-parsers/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-pcap-backend/target/ + + *.tar.gz + + + + ${project.basedir}/../../../../metron-platform/metron-solr/target/ + + *.tar.gz + + + + + + + copy-rpms-target + package + + copy-resources + + + ${project.basedir}/target/RPMS + + + RPMS + + + + + + + + + diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml index 7f9c013ab3..b412ed813c 100644 --- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml +++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml @@ -50,7 +50,7 @@ **/*.formatted **/*.filtered - 0755 + 0644 unix true diff --git a/pom.xml b/pom.xml index c25d97c19c..1a89a65d8c 100644 --- a/pom.xml +++ b/pom.xml @@ -203,7 +203,8 @@ metron-ui/lib/public/font/** metron-ui/node_modules/** - metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/*.p + **/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/*.p + **/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p From de5534a566a9a4db77ff0a6d4cf8b512ca4df1e2 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 30 Aug 2016 18:47:09 -0400 Subject: [PATCH 44/82] Fixing DDL perms --- .../metron-enrichment/src/main/assembly/assembly.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml index 7f9c013ab3..b412ed813c 100644 --- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml +++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml @@ -50,7 +50,7 @@ **/*.formatted **/*.filtered - 0755 + 0644 unix true From 6ecb4ccc351e07ddf122d1ca730fe1a2d6874782 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 30 Aug 2016 19:42:38 -0400 Subject: [PATCH 45/82] Need HBASE dependency for threat intel --- .../resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml index 3d01d76a8f..1ca1f279d3 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/metainfo.xml @@ -145,6 +145,7 @@ KAFKA STORM ZOOKEEPER + HBASE From f9c27f3a4645fdf7d01c2758c7272090ec975dcc Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 31 Aug 2016 07:58:34 -0400 Subject: [PATCH 46/82] Allow change of storm ui server location. --- .../METRON/0.2.0BETA/configuration/metron-env.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index 3c68bc9f16..a32746ec56 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -77,10 +77,10 @@ storm_rest_addr - - true - false - + + + + From fdf73eb27761e215644ec6a0c50c99f78ab94d27 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 31 Aug 2016 10:48:38 -0400 Subject: [PATCH 47/82] WIP HBase table creation --- .../0.2.0BETA/package/scripts/commands.py | 17 +++++++++++++++++ .../package/scripts/enrichment_master.py | 1 + .../0.2.0BETA/package/scripts/mysql_users.py | 1 - 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py index 86803b3794..3e011eba12 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py @@ -145,3 +145,20 @@ def is_topology_active(self): status = stdout.split()[1] running_status_set = {'ACTIVE', 'REBALANCING'} return status in running_status_set + + def create_hbase_tables(self): + add_enrichment_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") + Execute(add_enrichment_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) + + add_threatintel_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") + Execute(add_threatintel_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py index 9b24e061b3..18b826ea0f 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py @@ -37,6 +37,7 @@ def start(self, env, upgrade_type=None): if not commands.is_configured(): commands.init_kafka_topics() + commands.create_hbase_tables() commands.set_configured() commands.start_enrichment_topology() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py index 186bcb5c1f..ce23b81707 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py @@ -32,7 +32,6 @@ def mysql_adduser(): content=StaticFile('addMysqlUser.sh') ) - # TODO figure out password add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}") Execute(add_user_cmd, tries=3, From 0d484817bc3222f36eb96d653e508679eba8f415 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 31 Aug 2016 11:11:57 -0400 Subject: [PATCH 48/82] Changing config name --- .../ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml index 1d87adfc70..b6002e69e7 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml @@ -49,7 +49,7 @@ metron_enrichment_db_password PASSWORD - GeoIP Database Password + Metron Enrichment Database Password Password to use against database password From 6a2527790103c18bb86e55cc8ba749ecd4250112 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 31 Aug 2016 11:28:09 -0400 Subject: [PATCH 49/82] RCO + retries fixes --- .../ENRICHMENT/0.2.0BETA/package/scripts/commands.py | 4 ---- .../HDP/2.4/services/ENRICHMENT/role_command_order.json | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py index 3e011eba12..39b41a3deb 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py @@ -149,16 +149,12 @@ def is_topology_active(self): def create_hbase_tables(self): add_enrichment_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") Execute(add_enrichment_cmd, - tries=3, - try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) add_threatintel_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") Execute(add_threatintel_cmd, - tries=3, - try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json index d88e649a89..cb1d39c525 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ENRICHMENT/role_command_order.json @@ -3,7 +3,7 @@ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", "general_deps" : { "_comment" : "dependencies for all cases", - "ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START"], + "ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], "ENRICHMENT_MYSQL_SERVER-STOP" : ["ENRICHMENT_MASTER-STOP"], "ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["ENRICHMENT_MASTER-START"] } From 23580073bf66d7d2ed5a2fd056cc81f2aedf8d3e Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 31 Aug 2016 13:59:12 -0400 Subject: [PATCH 50/82] Retries fixes --- .../ENRICHMENT/0.2.0BETA/package/scripts/commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py index 39b41a3deb..2ccb9316e1 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py @@ -149,12 +149,16 @@ def is_topology_active(self): def create_hbase_tables(self): add_enrichment_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") Execute(add_enrichment_cmd, + tries=3, + try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) - add_threatintel_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") + add_threatintel_cmd = format("echo \"create 'threatintel','t'\" | hbase shell -n") Execute(add_threatintel_cmd, + tries=3, + try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) From b746a5a9b30350cdc4bf5393252f696c83463c27 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 31 Aug 2016 17:48:24 -0400 Subject: [PATCH 51/82] merge in enrichment --- .../METRON/0.2.0BETA/role_command_order.json | 3 + .../0.2.0BETA/configuration/metron-env.xml | 17 ++ .../METRON/0.2.0BETA/metainfo.xml | 91 +++++++++- .../0.2.0BETA/package/files/addMysqlUser.sh | 41 +++++ .../package/files/createMysqlGeoIp.sh | 40 +++++ .../package/files/removeMysqlUser.sh | 33 ++++ .../package/scripts/enrichment_commands.py | 164 ++++++++++++++++++ .../package/scripts/enrichment_master.py | 73 ++++++++ .../package/scripts/indexing_master.py | 13 +- .../0.2.0BETA/package/scripts/mysql_server.py | 61 +++++++ .../package/scripts/mysql_service.py | 46 +++++ .../0.2.0BETA/package/scripts/mysql_users.py | 58 +++++++ .../0.2.0BETA/package/scripts/mysql_utils.py | 50 ++++++ .../package/scripts/params/params_linux.py | 37 +++- .../package/scripts/params/status_params.py | 47 +++++ .../package/scripts/parser_master.py | 9 +- 16 files changed, 770 insertions(+), 13 deletions(-) create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json index 871062c133..0f4cd3914e 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json @@ -5,6 +5,9 @@ "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], + "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], + "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"], + "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"] } diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index 6d08e630bd..cdc52f767f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -60,6 +60,23 @@ Metron parsers to deploy Metron parsers + + metron_enrichment_db_user + root + Database username to use to connect to the database. + + + metron_enrichment_db_password + + PASSWORD + Metron Enrichment Database Password + Password to use against database + + password + false + + + metron_indexing_topology indexing diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml index 827cfef0e5..481e468b1f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -82,6 +82,65 @@ + + METRON_ENRICHMENT_MASTER + Enrichment Master + MASTER + 1 + true + + + HDFS/HDFS_CLIENT + host + + true + + + + ZOOKEEPER/ZOOKEEPER_SERVER + cluster + + true + + + + KAFKA/KAFKA_BROKER + cluster + + true + + + + + + PYTHON + 600 + + + + + METRON_ENRICHMENT_MYSQL_SERVER + MySQL Server + MASTER + 1 + false + + + + PYTHON + + + + CLEAN + + + PYTHON + 600 + + + + + METRON_INDEXING Metron Indexing @@ -122,7 +181,6 @@ - @@ -135,6 +193,9 @@ metron-parsers + + metron-enrichment + metron-indexing @@ -143,6 +204,33 @@ + + redhat7 + + + mysql-community-release + true + + + mysql-community-server + true + + MySQL-python + + + + redhat6 + + + mysql-server + true + + + mysql + true + + + @@ -156,6 +244,7 @@ KAFKA STORM ZOOKEEPER + HBASE diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh new file mode 100755 index 0000000000..c8c2d68636 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +mysqldbuser=$2 +mysqldbpasswd=$3 +mysqldbhost=$4 +myhostname=$(hostname -f) + +service $mysqldservice start +echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost" +mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';" +mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';" +mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';" +mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';" +if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then + echo "Adding user $mysqldbuser@$myhostname"; + mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';"; + mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$myhostname';"; +fi +mysql -u root -e "flush privileges;" +service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh new file mode 100755 index 0000000000..cbe14cdb24 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +geoipscript=$2 + +# Download and extract the actual GeoIP files +mkdir -p /tmp/geoip +mkdir -p /var/lib/mysql-files/ + +pushd /tmp/geoip +curl -O http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz +tar xf GeoLiteCity-latest.tar.xz +cp /tmp/geoip/*/*.csv /var/lib/mysql-files/ +popd + +# Load MySQL with the GeoIP data +service ${mysqldservice} start +mysql -u root < ${geoipscript} +mysql -u root -e "show databases;" +service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh new file mode 100755 index 0000000000..33a5ee5b0c --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +mysqldbuser=$2 +userhost=$3 +myhostname=$(hostname -f) +sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E" + +${sudo_prefix} service ${mysqldservice} start +echo "Removing user $mysqldbuser@$userhost" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\"" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\"" +${sudo_prefix} service ${mysqldservice} stop diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py new file mode 100755 index 0000000000..475c523c2a --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import subprocess +import time + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, File + + +# Wrap major operations and functionality in this class +class EnrichmentCommands: + __params = None + __enrichment = None + __configured = False + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__enrichment = params.metron_enrichment_topology + self.__configured = os.path.isfile(self.__params.configured_flag_file) + + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def setup_repo(self): + def local_repo(): + Logger.info("Setting up local repo") + Execute("yum -y install createrepo") + Execute("createrepo /localrepo") + Execute("chmod -R o-w+r /localrepo") + Execute("echo \"[METRON-0.2.0BETA]\n" + "name=Metron 0.2.0BETA packages\n" + "baseurl=file:///localrepo\n" + "gpgcheck=0\n" + "enabled=1\" > /etc/yum.repos.d/local.repo") + + def remote_repo(): + print('Using remote repo') + + yum_repo_types = { + 'local': local_repo, + 'remote': remote_repo + } + repo_type = self.__params.yum_repo_type + if repo_type in yum_repo_types: + yum_repo_types[repo_type]() + else: + raise ValueError("Unsupported repo type '{}'".format(repo_type)) + + def init_kafka_topics(self): + Logger.info('Creating Kafka topics') + command_template = """{}/kafka-topics.sh \ + --zookeeper {} \ + --create \ + --topic {} \ + --partitions {} \ + --replication-factor {} \ + --config retention.bytes={}""" + num_partitions = 1 + replication_factor = 1 + retention_gigabytes = 10 + retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 + Logger.info("Creating topics for enrichment") + + Logger.info("Creating topic'{}'".format(self.__enrichment)) + Execute(command_template.format(self.__params.kafka_bin_dir, + self.__params.zookeeper_quorum, + self.__enrichment, + num_partitions, + replication_factor, + retention_bytes)) + Logger.info("Done creating Kafka topics") + + def start_enrichment_topology(self): + Logger.info("Starting Metron enrichment topology: {}".format(self.__enrichment)) + start_cmd_template = """{}/bin/start_enrichment_topology.sh \ + -s {} \ + -z {}""" + Logger.info('Starting ' + self.__enrichment) + Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment, self.__params.zookeeper_quorum)) + + Logger.info('Finished starting enrichment topology') + + def stop_enrichment_topology(self): + Logger.info('Stopping ' + self.__enrichment) + stop_cmd = 'storm kill ' + self.__enrichment + Execute(stop_cmd) + Logger.info('Done stopping enrichment topologies') + + def restart_enrichment_topology(self): + Logger.info('Restarting the enrichment topologies') + self.stop_enrichment_topology() + + # Wait for old topology to be cleaned up by Storm, before starting again. + retries = 0 + topology_active = self.is_topology_active() + while topology_active and retries < 3: + Logger.info('Existing topology still active. Will wait and retry') + time.sleep(40) + topology_active = self.is_topology_active() + retries += 1 + + if not topology_active: + self.start_enrichment_topology() + Logger.info('Done restarting the enrichment topology') + else: + Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + + def is_topology_active(self): + cmd_retrieve = "storm list | grep 'enrichment'" + proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + Logger.info("Retrieval response is: %s" % stdout) + Logger.warning("Error response is: %s" % stderr) + + fields = stdout.split() + if len(fields) < 2: + Logger.warning("Enrichment topology is not running") + return False + + # Get the second column, which is status. We already know first column is enrichment) + status = stdout.split()[1] + running_status_set = {'ACTIVE', 'REBALANCING'} + return status in running_status_set + + def create_hbase_tables(self): + add_enrichment_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") + Execute(add_enrichment_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) + + add_threatintel_cmd = format("echo \"create 'threatintel','t'\" | hbase shell -n") + Execute(add_threatintel_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py new file mode 100755 index 0000000000..39b311931c --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py @@ -0,0 +1,73 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from enrichment_commands import EnrichmentCommands + +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.libraries.script import Script + + +class Enrichment(Script): + def install(self, env): + from params import params + env.set_params(params) + commands = EnrichmentCommands(params) + commands.setup_repo() + Logger.info('Install RPM packages') + self.install_packages(env) + + def start(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = EnrichmentCommands(params) + + if not commands.is_configured(): + commands.init_kafka_topics() + commands.create_hbase_tables() + commands.set_configured() + + commands.start_enrichment_topology() + + def stop(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = EnrichmentCommands(params) + commands.stop_enrichment_topology() + + def status(self, env): + from params import status_params + env.set_params(status_params) + commands = EnrichmentCommands(status_params) + + if not commands.is_topology_active(): + raise ComponentIsNotRunning() + + def restart(self, env): + from params import params + env.set_params(params) + commands = EnrichmentCommands(params) + commands.restart_enrichment_topology() + + def kafkabuild(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = EnrichmentCommands(params) + commands.init_kafka_topics() + + +if __name__ == "__main__": + Enrichment().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index 5a34c4f2e9..aa8511c721 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -17,11 +17,10 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from indexing_commands import IndexingCommands + import metron_service +from indexing_commands import IndexingCommands + class Indexing(Script): @@ -40,7 +39,7 @@ def configure(self, env): env.set_params(params) commands = IndexingCommands(params) - metron_service.init_config() + metron_service.load_global_config(params) if not commands.is_configured(): commands.init_kafka_topics() @@ -60,7 +59,7 @@ def stop(self, env, upgrade_type=None): commands.stop_indexing_topology() def status(self, env): - import status_params + from params import status_params env.set_params(status_params) commands = IndexingCommands(status_params) if not commands.is_topology_active(env): @@ -71,7 +70,7 @@ def restart(self, env): env.set_params(params) self.configure(env) commands = IndexingCommands(params) - commands.restart_indexing_topology() + commands.restart_indexing_topology(env) if __name__ == "__main__": diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py new file mode 100755 index 0000000000..2fc363fcc4 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script + +import mysql_users +from mysql_service import mysql_service +from mysql_utils import mysql_configure + + +class MysqlServer(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def clean(self, env): + from params import params + env.set_params(params) + mysql_users.mysql_deluser() + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) + mysql_configure() + + def start(self, env, rolling_restart=False): + from params import params + env.set_params(params) + mysql_service(daemon_name=params.daemon_name, action='start') + + def stop(self, env, rolling_restart=False): + from params import params + env.set_params(params) + mysql_service(daemon_name=params.daemon_name, action='stop') + + def status(self, env): + from params import status_params + env.set_params(status_params) + + mysql_service(daemon_name=status_params.daemon_name, action='status') + + +if __name__ == "__main__": + MysqlServer().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py new file mode 100755 index 0000000000..7c4a7e881a --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import ComponentIsNotRunning, Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.format import format + + +def mysql_service(daemon_name=None, action='start'): + status_cmd = format("pgrep -l '^{process_name}$'") + cmd = ('service', daemon_name, action) + + if action == 'status': + try: + Execute(status_cmd) + except Fail: + raise ComponentIsNotRunning() + elif action == 'stop': + Execute(cmd, + logoutput=True, + only_if=status_cmd, + sudo=True, + ) + elif action == 'start': + Execute(cmd, + logoutput=True, + not_if=status_cmd, + sudo=True, + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py new file mode 100755 index 0000000000..1721eba6f4 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import StaticFile +from resource_management.libraries.functions.format import format + + +# Used to add metron access to the needed components +def mysql_adduser(): + from params import params + + File(params.mysql_adduser_path, + mode=0755, + content=StaticFile('addMysqlUser.sh') + ) + + add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}") + Execute(add_user_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' + ) + + +# Removes hive metron from components +def mysql_deluser(): + from params import params + + File(params.mysql_deluser_path, + mode=0755, + content=StaticFile('removeMysqlUser.sh') + ) + + del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}") + Execute(del_user_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py new file mode 100755 index 0000000000..4cb522194a --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import StaticFile +from resource_management.libraries.functions.format import format + +import mysql_users + + +def mysql_configure(): + from params import params + + # required for running hive + replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname) + Execute(replace_bind_address, + sudo=True, + ) + + # this also will start mysql-server + mysql_users.mysql_adduser() + + File(params.mysql_create_geoip_path, + mode=0755, + content=StaticFile('createMysqlGeoIp.sh') + ) + + geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geo_ip_ddl}") + Execute(geoip_setup_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 77c3c06898..deec02cbce 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -21,28 +21,39 @@ import functools import os +from ambari_commons.os_check import OSCheck from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import format from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.script import Script +import status_params + # server configurations config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + hostname = config['hostname'] metron_home = config['configurations']['metron-env']['metron_home'] parsers = config['configurations']['metron-env']['parsers'] +metron_ddl_dir = metron_home + '/ddl' +geo_ip_ddl = metron_ddl_dir + '/geoip_ddl.sql' +metron_enrichment_topology = status_params.metron_enrichment_topology metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] metron_user = config['configurations']['metron-env']['metron_user'] metron_group = config['configurations']['metron-env']['metron_group'] metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' -indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' +enrichment_configured_flag_file = metron_zookeeper_config_path + '/metron_enrichment_is_configured' +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' global_json_template = config['configurations']['metron-env']['global-json'] +global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] es_url = config['configurations']['metron-env']['es_url'] yum_repo_type = 'local' @@ -68,7 +79,7 @@ # last port config zookeeper_quorum += ':' + zookeeper_clientPort -#Storm +# Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] # Kafka @@ -101,6 +112,28 @@ default_fs = config['configurations']['core-site']['fs.defaultFS'] dfs_type = default("/commandParams/dfs_type", "") +# MYSQL +if OSCheck.is_ubuntu_family(): + mysql_configname = '/etc/mysql/my.cnf' +else: + mysql_configname = '/etc/my.cnf' + +daemon_name = status_params.daemon_name +mysql_user = "mysql" +mysql_group = 'mysql' +mysql_host = config['clusterHostInfo']['enrichment_mysql_host'] + +mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" +mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" +mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh" + +enrichment_hosts = default("/clusterHostInfo/enrichment_host", []) +enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None +enrichment_metron_user_passwd = config['configurations']['metron-enrichment']['metron_enrichment_db_password'] +enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty( + enrichment_metron_user_passwd) else enrichment_metron_user_passwd +mysql_process_name = status_params.mysql_process_name + # create partial functions with common arguments for every HdfsResource call # to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py new file mode 100644 index 0000000000..fd6fc060d6 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons import OSCheck +from resource_management.libraries.functions import format +from resource_management.libraries.script import Script + +config = Script.get_config() + +# Parsers +parsers = config['configurations']['metron-env']['parsers'] +metron_home = config['configurations']['metron-env']['metron_home'] +metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] +metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') +parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' + +# Enrichment +metron_enrichment_topology = 'enrichment' +mysql_process_name = 'mysqld' +if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): + daemon_name = 'mysql' +else: + daemon_name = 'mysqld' + +# Indexing +metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] +indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' + +# Storm +storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index 7bba256bf8..eb90bc6630 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -20,8 +20,10 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.script import Script -from parser_commands import ParserCommands + import metron_service +from parser_commands import ParserCommands + class ParserMaster(Script): def get_component_name(self): @@ -62,7 +64,7 @@ def stop(self, env, upgrade_type=None): commands.stop_parser_topologies() def status(self, env): - import status_params + from params import status_params env.set_params(status_params) commands = ParserCommands(status_params) if not commands.topologies_running(env): @@ -71,8 +73,9 @@ def status(self, env): def restart(self, env): from params import params env.set_params(params) + self.configure(env) commands = ParserCommands(params) - commands.restart_parser_topologies() + commands.restart_parser_topologies(env) def servicechecktest(self,env): from params import params From 32d2f1c936ddff0a436383c1925d74d7d8053b93 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 1 Sep 2016 13:56:04 -0400 Subject: [PATCH 52/82] Configuring enrichment.properties appropriately --- .../configuration/metron-enrichment.xml | 10 +-- .../package/scripts/enrichment_master.py | 14 ++++ .../0.2.0BETA/package/scripts/params_linux.py | 9 +- .../templates/enrichment.properties.j2 | 82 +++++++++++++++++++ 4 files changed, 107 insertions(+), 8 deletions(-) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/templates/enrichment.properties.j2 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml index b6002e69e7..05cb16431c 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/configuration/metron-enrichment.xml @@ -36,14 +36,14 @@ Metron User - metron_group + metron_enrichment_db_user metron - The group for Metron + Database username to use to connect to the database. - metron_enrichment_db_user - root - Database username to use to connect to the database. + metron_enrichment_db_port + 3306 + Database port to use to connect to the database. metron_enrichment_db_password diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py index 18b826ea0f..cba227b384 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py @@ -16,7 +16,10 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger +from resource_management.core.resources.system import File from resource_management.libraries.script import Script +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format from commands import Commands @@ -29,6 +32,17 @@ def install(self, env): commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) + self.configure(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + + File(format("{metron_config_path}/enrichment.properties"), + content=Template("enrichment.properties.j2"), + owner=params.metron_user, + group=params.user_group + ) def start(self, env, upgrade_type=None): import params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py index 3e132a0774..d947172eb8 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py @@ -82,9 +82,9 @@ mysql_configname = '/etc/my.cnf' daemon_name = status_params.daemon_name -mysql_user = "mysql" -mysql_group = 'mysql' -mysql_host = config['clusterHostInfo']['enrichment_mysql_host'] +# There will always be exactly one mysql_host +mysql_host = config['clusterHostInfo']['enrichment_mysql_server_hosts'][0] +mysql_port = config['configurations']['metron-enrichment']['metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" @@ -94,6 +94,9 @@ enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None metron_user = config['configurations']['metron-enrichment']['metron_user'] +user_group = config['configurations']['cluster-env']['user_group'] + +enrichment_metron_user = config['configurations']['metron-enrichment']['metron_enrichment_db_user'] enrichment_metron_user_passwd = config['configurations']['metron-enrichment']['metron_enrichment_db_password'] enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(enrichment_metron_user_passwd) else enrichment_metron_user_passwd process_name = status_params.process_name diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/templates/enrichment.properties.j2 new file mode 100755 index 0000000000..e993ac99ef --- /dev/null +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/templates/enrichment.properties.j2 @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +##### Kafka ##### + +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +enrichment.output.topic=indexing + +##### MySQL ##### + +mysql.ip={{mysql_host}} +mysql.port={{mysql_port}} +mysql.username={{enrichment_metron_user}} +mysql.password={{enrichment_metron_user_passwd}} + +##### Metrics ##### + +#reporters +org.apache.metron.metrics.reporter.graphite=true +org.apache.metron.metrics.reporter.console=false +org.apache.metron.metrics.reporter.jmx=false + +#Graphite Addresses + +org.apache.metron.metrics.graphite.address=localhost +org.apache.metron.metrics.graphite.port=2023 + +#TelemetryParserBolt +org.apache.metron.metrics.TelemetryParserBolt.acks=true +org.apache.metron.metrics.TelemetryParserBolt.emits=true +org.apache.metron.metrics.TelemetryParserBolt.fails=true + + +#GenericEnrichmentBolt +org.apache.metron.metrics.GenericEnrichmentBolt.acks=true +org.apache.metron.metrics.GenericEnrichmentBolt.emits=true +org.apache.metron.metrics.GenericEnrichmentBolt.fails=true + + +#TelemetryIndexingBolt +org.apache.metron.metrics.TelemetryIndexingBolt.acks=true +org.apache.metron.metrics.TelemetryIndexingBolt.emits=true +org.apache.metron.metrics.TelemetryIndexingBolt.fails=true + +##### Host Enrichment ##### + +org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ +{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + + +##### HBase ##### +bolt.hbase.table.name=pcap +bolt.hbase.table.fields=t:value +bolt.hbase.table.key.tuple.field.name=key +bolt.hbase.table.timestamp.tuple.field.name=timestamp +bolt.hbase.enable.batching=false +bolt.hbase.write.buffer.size.in.bytes=2000000 +bolt.hbase.durability=SKIP_WAL +bolt.hbase.partitioner.region.info.refresh.interval.mins=60 + +##### Threat Intel ##### + +threat.intel.tracker.table=threatintel +threat.intel.tracker.cf=t +threat.intel.ip.table= +threat.intel.ip.cf= From 45157ab17bfc8cf31b766a86d72f70c7ae24e689 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 1 Sep 2016 14:32:44 -0400 Subject: [PATCH 53/82] fix param ref --- .../METRON/0.2.0BETA/package/scripts/enrichment_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 475c523c2a..5fa27c522e 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -34,13 +34,13 @@ def __init__(self, params): raise ValueError("params argument is required for initialization") self.__params = params self.__enrichment = params.metron_enrichment_topology - self.__configured = os.path.isfile(self.__params.configured_flag_file) + self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) def is_configured(self): return self.__configured def set_configured(self): - File(self.__params.configured_flag_file, + File(self.__params.enrichment_configured_flag_file, content="", owner=self.__params.metron_user, mode=0775) From 22b63d5dff1cbd3a672790a56f2046ed0ba4059e Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 1 Sep 2016 16:07:10 -0400 Subject: [PATCH 54/82] Some cleanup and renaming --- .../{commands.py => enrichment_commands.py} | 2 +- .../package/scripts/enrichment_master.py | 30 +++++++++---------- .../0.2.0BETA/package/scripts/mysql_server.py | 10 +++---- .../0.2.0BETA/package/scripts/mysql_users.py | 4 +-- .../0.2.0BETA/package/scripts/mysql_utils.py | 2 +- .../package/scripts/params/__init__.py | 0 .../package/scripts/{ => params}/params.py | 0 .../scripts/{ => params}/params_linux.py | 2 +- .../scripts/{ => params}/params_windows.py | 0 .../scripts/{ => params}/status_params.py | 0 .../package/scripts/service_check.py | 6 ++-- 11 files changed, 28 insertions(+), 28 deletions(-) rename metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/{commands.py => enrichment_commands.py} (99%) create mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py rename metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/{ => params}/params.py (100%) rename metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/{ => params}/params_linux.py (100%) rename metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/{ => params}/params_windows.py (100%) rename metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/{ => params}/status_params.py (100%) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_commands.py similarity index 99% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_commands.py index 2ccb9316e1..475c523c2a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/commands.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_commands.py @@ -24,7 +24,7 @@ # Wrap major operations and functionality in this class -class Commands: +class EnrichmentCommands: __params = None __enrichment = None __configured = False diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py index cba227b384..45661adab6 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/enrichment_master.py @@ -17,25 +17,25 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import File -from resource_management.libraries.script import Script from resource_management.core.source import Template from resource_management.libraries.functions.format import format +from resource_management.libraries.script import Script -from commands import Commands +from enrichment_commands import EnrichmentCommands class Enrichment(Script): def install(self, env): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) self.configure(env) def configure(self, env, upgrade_type=None, config_dir=None): - import params + from params import params env.set_params(params) File(format("{metron_config_path}/enrichment.properties"), @@ -45,9 +45,9 @@ def configure(self, env, upgrade_type=None, config_dir=None): ) def start(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) if not commands.is_configured(): commands.init_kafka_topics() @@ -57,29 +57,29 @@ def start(self, env, upgrade_type=None): commands.start_enrichment_topology() def stop(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) commands.stop_enrichment_topology() def status(self, env): - import status_params + from params import status_params env.set_params(status_params) - commands = Commands(status_params) + commands = EnrichmentCommands(status_params) if not commands.is_topology_active(): raise ComponentIsNotRunning() def restart(self, env): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) commands.restart_enrichment_topology() def kafkabuild(self, env, upgrade_type=None): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) commands.init_kafka_topics() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py index 2241580fcc..2fc363fcc4 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_server.py @@ -31,27 +31,27 @@ def install(self, env): self.configure(env) def clean(self, env): - import params + from params import params env.set_params(params) mysql_users.mysql_deluser() def configure(self, env, upgrade_type=None, config_dir=None): - import params + from params import params env.set_params(params) mysql_configure() def start(self, env, rolling_restart=False): - import params + from params import params env.set_params(params) mysql_service(daemon_name=params.daemon_name, action='start') def stop(self, env, rolling_restart=False): - import params + from params import params env.set_params(params) mysql_service(daemon_name=params.daemon_name, action='stop') def status(self, env): - import status_params + from params import status_params env.set_params(status_params) mysql_service(daemon_name=status_params.daemon_name, action='status') diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py index ce23b81707..1721eba6f4 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_users.py @@ -25,7 +25,7 @@ # Used to add metron access to the needed components def mysql_adduser(): - import params + from params import params File(params.mysql_adduser_path, mode=0755, @@ -43,7 +43,7 @@ def mysql_adduser(): # Removes hive metron from components def mysql_deluser(): - import params + from params import params File(params.mysql_deluser_path, mode=0755, diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py index 470fe551a7..4cb522194a 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/mysql_utils.py @@ -26,7 +26,7 @@ def mysql_configure(): - import params + from params import params # required for running hive replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params_linux.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params_linux.py index d947172eb8..ec3cbfd323 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_linux.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params_linux.py @@ -21,8 +21,8 @@ from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.default import default -from resource_management.libraries.script import Script from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.script import Script import status_params diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params_windows.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params_windows.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/params_windows.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/status_params.py similarity index 100% rename from metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/status_params.py rename to metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/status_params.py diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py index c02d695919..35c723071b 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/service_check.py @@ -18,15 +18,15 @@ from resource_management.libraries.script import Script -from commands import Commands +from enrichment_commands import EnrichmentCommands class ServiceCheck(Script): def service_check(self, env): - import params + from params import params env.set_params(params) - commands = Commands(params) + commands = EnrichmentCommands(params) if commands.is_topology_active(): exit(0) else: From 15843af6accc70e8b00fc3f97706a42b4a87965a Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 1 Sep 2016 16:07:37 -0400 Subject: [PATCH 55/82] Merge Justin's changes --- .../0.2.0BETA/configuration/metron-env.xml | 7 +- .../package/scripts/enrichment_master.py | 18 +++- .../package/scripts/params/params_linux.py | 8 +- .../package/scripts/status_params.py | 39 --------- .../templates/enrichment.properties.j2 | 82 +++++++++++++++++++ 5 files changed, 109 insertions(+), 45 deletions(-) delete mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py create mode 100755 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index 0212d553f5..a86cf20b36 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -62,9 +62,14 @@ metron_enrichment_db_user - root + metron Database username to use to connect to the database. + + metron_enrichment_db_port + 3306 + Database port to use to connect to the database. + metron_enrichment_db_password diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py index 39b311931c..b589186297 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py @@ -14,11 +14,14 @@ limitations under the License. """ -from enrichment_commands import EnrichmentCommands - from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger +from resource_management.core.resources.system import File from resource_management.libraries.script import Script +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format + +from enrichment_commands import EnrichmentCommands class Enrichment(Script): @@ -29,6 +32,17 @@ def install(self, env): commands.setup_repo() Logger.info('Install RPM packages') self.install_packages(env) + self.configure(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) + + File(format("{metron_config_path}/enrichment.properties"), + content=Template("enrichment.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) def start(self, env, upgrade_type=None): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index deec02cbce..91cba8d93a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -119,9 +119,9 @@ mysql_configname = '/etc/my.cnf' daemon_name = status_params.daemon_name -mysql_user = "mysql" -mysql_group = 'mysql' -mysql_host = config['clusterHostInfo']['enrichment_mysql_host'] +# There will always be exactly one mysql_host +mysql_host = config['clusterHostInfo']['enrichment_mysql_server_hosts'][0] +mysql_port = config['configurations']['metron-enrichment']['metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" @@ -129,6 +129,8 @@ enrichment_hosts = default("/clusterHostInfo/enrichment_host", []) enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None + +enrichment_metron_user = config['configurations']['metron-enrichment']['metron_enrichment_db_user'] enrichment_metron_user_passwd = config['configurations']['metron-enrichment']['metron_enrichment_db_password'] enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty( enrichment_metron_user_passwd) else enrichment_metron_user_passwd diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py deleted file mode 100644 index d14fc24237..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/status_params.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.functions import format -from resource_management.libraries.functions.default import default -from resource_management.libraries.script import Script - - -config = Script.get_config() -#Parsers -parsers = config['configurations']['metron-env']['parsers'] -metron_home = config['configurations']['metron-env']['metron_home'] -metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' - -#Indexing -metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] -indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' - -#Storm -storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 new file mode 100755 index 0000000000..e993ac99ef --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +##### Kafka ##### + +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +enrichment.output.topic=indexing + +##### MySQL ##### + +mysql.ip={{mysql_host}} +mysql.port={{mysql_port}} +mysql.username={{enrichment_metron_user}} +mysql.password={{enrichment_metron_user_passwd}} + +##### Metrics ##### + +#reporters +org.apache.metron.metrics.reporter.graphite=true +org.apache.metron.metrics.reporter.console=false +org.apache.metron.metrics.reporter.jmx=false + +#Graphite Addresses + +org.apache.metron.metrics.graphite.address=localhost +org.apache.metron.metrics.graphite.port=2023 + +#TelemetryParserBolt +org.apache.metron.metrics.TelemetryParserBolt.acks=true +org.apache.metron.metrics.TelemetryParserBolt.emits=true +org.apache.metron.metrics.TelemetryParserBolt.fails=true + + +#GenericEnrichmentBolt +org.apache.metron.metrics.GenericEnrichmentBolt.acks=true +org.apache.metron.metrics.GenericEnrichmentBolt.emits=true +org.apache.metron.metrics.GenericEnrichmentBolt.fails=true + + +#TelemetryIndexingBolt +org.apache.metron.metrics.TelemetryIndexingBolt.acks=true +org.apache.metron.metrics.TelemetryIndexingBolt.emits=true +org.apache.metron.metrics.TelemetryIndexingBolt.fails=true + +##### Host Enrichment ##### + +org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ +{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + + +##### HBase ##### +bolt.hbase.table.name=pcap +bolt.hbase.table.fields=t:value +bolt.hbase.table.key.tuple.field.name=key +bolt.hbase.table.timestamp.tuple.field.name=timestamp +bolt.hbase.enable.batching=false +bolt.hbase.write.buffer.size.in.bytes=2000000 +bolt.hbase.durability=SKIP_WAL +bolt.hbase.partitioner.region.info.refresh.interval.mins=60 + +##### Threat Intel ##### + +threat.intel.tracker.table=threatintel +threat.intel.tracker.cf=t +threat.intel.ip.table= +threat.intel.ip.cf= From 48150777aa726dc91cf57b84665ec1e815729a34 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 2 Sep 2016 08:57:10 -0400 Subject: [PATCH 56/82] Fixing license issue --- .../0.2.0BETA/package/scripts/params/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py index e69de29bb2..d9e26de2ad 100755 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py +++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/ENRICHMENT/0.2.0BETA/package/scripts/params/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From 4600c14d6c884d7f5204d1217eee1f881395d9f7 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 6 Sep 2016 08:16:06 -0400 Subject: [PATCH 57/82] Updating with various config fixes --- .../package/scripts/enrichment_commands.py | 5 ++-- .../package/scripts/enrichment_master.py | 2 +- .../package/scripts/indexing_commands.py | 21 +++++++-------- .../package/scripts/mysql_service.py | 2 +- .../package/scripts/params/params_linux.py | 27 ++++++++++--------- .../package/scripts/params/status_params.py | 3 +++ 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 5fa27c522e..cdf3f90602 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -129,7 +129,8 @@ def restart_enrichment_topology(self): else: Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') - def is_topology_active(self): + def is_topology_active(self, env): + env.set_params(self.__params) cmd_retrieve = "storm list | grep 'enrichment'" proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() @@ -143,7 +144,7 @@ def is_topology_active(self): # Get the second column, which is status. We already know first column is enrichment) status = stdout.split()[1] - running_status_set = {'ACTIVE', 'REBALANCING'} + running_status_set = ['ACTIVE', 'REBALANCING'] return status in running_status_set def create_hbase_tables(self): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py index b589186297..8e105cb255 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py @@ -67,7 +67,7 @@ def status(self, env): env.set_params(status_params) commands = EnrichmentCommands(status_params) - if not commands.is_topology_active(): + if not commands.is_topology_active(env): raise ComponentIsNotRunning() def restart(self, env): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 3817564309..dbe21cefc4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -42,11 +42,11 @@ def __init__(self, params): def is_configured(self): return self.__configured - def set_configured(self): - File(self.__params.parsers_configured_flag_file, - content="", - owner=self.__params.metron_user, - mode=0775) + # def set_configured(self): + # File(self.__params.parsers_configured_flag_file, + # content="", + # owner=self.__params.metron_user, + # mode=0775) def setup_repo(self): def local_repo(): @@ -113,7 +113,7 @@ def stop_indexing_topology(self): Execute(stop_cmd) Logger.info('Done stopping indexing topologies') - def restart_indexing_topology(self,env): + def restart_indexing_topology(self, env): Logger.info('Restarting the indexing topologies') self.stop_indexing_topology() @@ -133,9 +133,8 @@ def restart_indexing_topology(self,env): else: Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') - - def is_configured(self): - return self.__configured + # def is_configured(self): + # return self.__configured def set_configured(self): File(self.__params.indexing_configured_flag_file, @@ -143,12 +142,12 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) - def is_topology_active(self,env): + def is_topology_active(self, env): env.set_params(self.__params) active = True topologies = metron_service.get_running_topologies() is_running = False if 'indexing' in topologies: - is_running = topologies['indexing'] in ['ACTIVE','REBALANCING'] + is_running = topologies['indexing'] in ['ACTIVE', 'REBALANCING'] active &= is_running return active diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py index 7c4a7e881a..2e0ce8b48f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py @@ -24,7 +24,7 @@ def mysql_service(daemon_name=None, action='start'): - status_cmd = format("pgrep -l '^{process_name}$'") + status_cmd = format("pgrep -l '^{mysql_process_name}$'") cmd = ('service', daemon_name, action) if action == 'status': diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 91cba8d93a..c5e4eda365 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -39,19 +39,20 @@ tmp_dir = Script.get_tmp_dir() hostname = config['hostname'] -metron_home = config['configurations']['metron-env']['metron_home'] -parsers = config['configurations']['metron-env']['parsers'] +metron_home = status_params.metron_home +parsers = status_params.parsers metron_ddl_dir = metron_home + '/ddl' geo_ip_ddl = metron_ddl_dir + '/geoip_ddl.sql' metron_enrichment_topology = status_params.metron_enrichment_topology -metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] +metron_indexing_topology = status_params.metron_indexing_topology metron_user = config['configurations']['metron-env']['metron_user'] metron_group = config['configurations']['metron-env']['metron_group'] -metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] -metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') -parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' -enrichment_configured_flag_file = metron_zookeeper_config_path + '/metron_enrichment_is_configured' -indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' +metron_config_path = metron_home + '/config' +metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir +metron_zookeeper_config_path = status_params.metron_zookeeper_config_path +parsers_configured_flag_file = status_params.parsers_configured_flag_file +enrichment_configured_flag_file = status_params.enrichment_configured_flag_file +indexing_configured_flag_file = status_params.indexing_configured_flag_file global_json_template = config['configurations']['metron-env']['global-json'] global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] @@ -80,7 +81,7 @@ zookeeper_quorum += ':' + zookeeper_clientPort # Storm -storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] +storm_rest_addr = status_params.storm_rest_addr # Kafka kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", []) @@ -120,8 +121,8 @@ daemon_name = status_params.daemon_name # There will always be exactly one mysql_host -mysql_host = config['clusterHostInfo']['enrichment_mysql_server_hosts'][0] -mysql_port = config['configurations']['metron-enrichment']['metron_enrichment_db_port'] +mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0] +mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" @@ -130,8 +131,8 @@ enrichment_hosts = default("/clusterHostInfo/enrichment_host", []) enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None -enrichment_metron_user = config['configurations']['metron-enrichment']['metron_enrichment_db_user'] -enrichment_metron_user_passwd = config['configurations']['metron-enrichment']['metron_enrichment_db_password'] +enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user'] +enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password'] enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty( enrichment_metron_user_passwd) else enrichment_metron_user_passwd mysql_process_name = status_params.mysql_process_name diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py index fd6fc060d6..d0afb9dcf8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py @@ -43,5 +43,8 @@ metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' +# Enrichment +enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_is_configured' + # Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] From 4e39e3cffc46a1905a019be77823228acd1ef1cd Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 6 Sep 2016 08:25:17 -0400 Subject: [PATCH 58/82] Slight code reorg --- .../package/scripts/indexing_commands.py | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index dbe21cefc4..95dc9baf1c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -15,15 +15,12 @@ limitations under the License. """ -import subprocess +import os import time - from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File -from resource_management.libraries.functions import format import metron_service -import os # Wrap major operations and functionality in this class @@ -42,11 +39,11 @@ def __init__(self, params): def is_configured(self): return self.__configured - # def set_configured(self): - # File(self.__params.parsers_configured_flag_file, - # content="", - # owner=self.__params.metron_user, - # mode=0775) + def set_configured(self): + File(self.__params.indexing_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) def setup_repo(self): def local_repo(): @@ -133,15 +130,6 @@ def restart_indexing_topology(self, env): else: Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') - # def is_configured(self): - # return self.__configured - - def set_configured(self): - File(self.__params.indexing_configured_flag_file, - content="", - owner=self.__params.metron_user, - mode=0775) - def is_topology_active(self, env): env.set_params(self.__params) active = True From 1ca84576dab421ca1b3051fb06cbffe9a37e3ac9 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Mon, 12 Sep 2016 10:53:22 -0400 Subject: [PATCH 59/82] merge cleanup from justin into mpack dir --- .../0.2.0BETA/package/scripts/indexing_commands.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 95dc9baf1c..82b73776bf 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -36,15 +36,6 @@ def __init__(self, params): self.__indexing = params.metron_indexing_topology self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file) - def is_configured(self): - return self.__configured - - def set_configured(self): - File(self.__params.indexing_configured_flag_file, - content="", - owner=self.__params.metron_user, - mode=0775) - def setup_repo(self): def local_repo(): Logger.info("Setting up local repo") From efde17eb88087d67146bf57463073050d174b74b Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 13 Sep 2016 08:56:16 -0400 Subject: [PATCH 60/82] Fixing config issues and a couple function calls --- .../METRON/0.2.0BETA/package/scripts/enrichment_commands.py | 6 +++--- .../0.2.0BETA/package/templates/enrichment.properties.j2 | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index cdf3f90602..798da1c237 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -110,17 +110,17 @@ def stop_enrichment_topology(self): Execute(stop_cmd) Logger.info('Done stopping enrichment topologies') - def restart_enrichment_topology(self): + def restart_enrichment_topology(self, env): Logger.info('Restarting the enrichment topologies') self.stop_enrichment_topology() # Wait for old topology to be cleaned up by Storm, before starting again. retries = 0 - topology_active = self.is_topology_active() + topology_active = self.is_topology_active(env) while topology_active and retries < 3: Logger.info('Existing topology still active. Will wait and retry') time.sleep(40) - topology_active = self.is_topology_active() + topology_active = self.is_topology_active(env) retries += 1 if not topology_active: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 index e993ac99ef..cc4b4d20ec 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 @@ -59,6 +59,9 @@ org.apache.metron.metrics.TelemetryIndexingBolt.fails=true ##### Host Enrichment ##### +##### hbase.provider.impl=org.apache.metron.hbase.HTableProvider ##### +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] @@ -78,5 +81,8 @@ bolt.hbase.partitioner.region.info.refresh.interval.mins=60 threat.intel.tracker.table=threatintel threat.intel.tracker.cf=t +threat.intel.simple.hbase.table=threatintel +threat.intel.simple.hbase.cf=t threat.intel.ip.table= threat.intel.ip.cf= + From 32307805206ed76ec2b9a28204d259b8729e3f4e Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 13 Sep 2016 12:32:18 -0400 Subject: [PATCH 61/82] Fixes to kafka topic names, along with further config fixes, and a mysql fix to allow storm to connect remotely --- .../METRON/0.2.0BETA/package/files/addMysqlUser.sh | 3 +++ .../METRON/0.2.0BETA/package/scripts/params/status_params.py | 2 +- .../0.2.0BETA/package/templates/enrichment.properties.j2 | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh index c8c2d68636..5dafcb87d4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh @@ -30,8 +30,11 @@ service $mysqldservice start echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost" mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';" mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';" + mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';" mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';" +mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'%' IDENTIFIED BY '$mysqldbpasswd';" + if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then echo "Adding user $mysqldbuser@$myhostname"; mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';"; diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py index d0afb9dcf8..498ffea7e2 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py @@ -32,7 +32,7 @@ parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' # Enrichment -metron_enrichment_topology = 'enrichment' +metron_enrichment_topology = 'enrichments' mysql_process_name = 'mysqld' if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): daemon_name = 'mysql' diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 index cc4b4d20ec..fad8530003 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 @@ -59,7 +59,7 @@ org.apache.metron.metrics.TelemetryIndexingBolt.fails=true ##### Host Enrichment ##### -##### hbase.provider.impl=org.apache.metron.hbase.HTableProvider ##### +hbase.provider.impl=org.apache.metron.hbase.HTableProvider enrichment.simple.hbase.table=enrichment enrichment.simple.hbase.cf=t org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ From 1965c7f7cac30aa24531adcea9994f4fde190198 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 13 Sep 2016 15:45:57 -0400 Subject: [PATCH 62/82] Adding some logging to bolts --- .../main/java/org/apache/metron/enrichment/bolt/JoinBolt.java | 1 + .../org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java | 1 + .../org/apache/metron/writer/bolt/BulkMessageWriterBolt.java | 1 + 3 files changed, 3 insertions(+) diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java index 2eb4b0abc0..e0c96a2976 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java @@ -115,6 +115,7 @@ public void execute(Tuple tuple) { ); cache.invalidate(key); collector.ack(tuple); + LOG.trace("Emitted message for key: {}", key); } else { cache.put(key, streamMessageMap); if(LOG.isDebugEnabled()) { diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 00d0498b7a..672d1356fe 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -90,6 +90,7 @@ public Map getFieldMap(String sourceType) { @Override public JSONObject joinMessages(Map streamMessageMap) { JSONObject ret = super.joinMessages(streamMessageMap); + LOG.trace("Received joined messages: {}", ret); boolean isAlert = ret.containsKey("is_alert"); if(!isAlert) { for (Object key : ret.keySet()) { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java index 91e8446a42..c4f41500fa 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java @@ -101,6 +101,7 @@ public void execute(Tuple tuple) { , bulkMessageWriter , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations())) ); + LOG.trace("Writing enrichment message: {}", message); } catch(Exception e) { throw new RuntimeException("This should have been caught in the writerComponent. If you see this, file a JIRA", e); From c7f6660454e9dbff82643facedbc25172e1cd43f Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 13 Sep 2016 17:18:45 -0400 Subject: [PATCH 63/82] Fixing indexing config check --- .../0.2.0BETA/package/scripts/indexing_commands.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 82b73776bf..95dc9baf1c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -36,6 +36,15 @@ def __init__(self, params): self.__indexing = params.metron_indexing_topology self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file) + def is_configured(self): + return self.__configured + + def set_configured(self): + File(self.__params.indexing_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + def setup_repo(self): def local_repo(): Logger.info("Setting up local repo") From 5babd620a78e5d92aab1c81fead7019125c362bb Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 09:44:51 -0400 Subject: [PATCH 64/82] Logging updates to make debugging easier --- .../apache/metron/enrichment/adapters/geo/GeoAdapter.java | 5 +++-- .../adapters/simplehbase/SimpleHBaseAdapter.java | 1 + .../enrichment/adapters/stellar/StellarAdapter.java | 8 +++++++- .../adapters/threatintel/ThreatIntelAdapter.java | 5 +++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java index 5b1bf9bc41..e10af77b21 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java @@ -45,7 +45,7 @@ public String getOutputPrefix(CacheKey value) { public JSONObject enrich(CacheKey value) { JSONObject enriched = new JSONObject(); if(!resetConnectionIfNecessary()) { - _LOG.error("Enrichment failure, cannot maintain a connection to JDBC. Please check connection. In the meantime, I'm not enriching."); + _LOG.error("GEO Enrichment failure, cannot maintain a connection to JDBC. Please check connection. In the meantime, I'm not enriching."); return enriched; } try { @@ -78,9 +78,10 @@ public JSONObject enrich(CacheKey value) { } resultSet.close(); } catch (Exception e) { - _LOG.error("Enrichment failure: " + e.getMessage(), e); + _LOG.error("GEO Enrichment failure: " + e.getMessage(), e); return new JSONObject(); } + _LOG.trace("GEO Enrichment success: ", enriched); return enriched; } } diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java index d2af3ccb42..240531c7f2 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java @@ -99,6 +99,7 @@ public JSONObject enrich(CacheKey value) { throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e); } } + _LOG.trace("SimpleHBaseAdapter succeeded:", enriched); return enriched; } diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java index 3ba7755d8f..c7bb32a3cd 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java @@ -27,6 +27,8 @@ import org.apache.metron.enrichment.bolt.CacheKey; import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.Map; @@ -35,6 +37,7 @@ import static org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.STELLAR_CONTEXT_CONF; public class StellarAdapter implements EnrichmentAdapter,Serializable { + protected static final Logger _LOG = LoggerFactory.getLogger(StellarAdapter.class); private enum EnrichmentType implements Function{ ENRICHMENT(config -> config.getEnrichment().getEnrichmentConfigs().get("stellar")) @@ -78,6 +81,7 @@ public JSONObject enrich(CacheKey value) { Map globalConfig = value.getConfig().getConfiguration(); Map sensorConfig = value.getConfig().getEnrichment().getConfig(); if(handler == null) { + _LOG.trace("Stellar ConfigHandler is null."); return new JSONObject(); } Map message = value.getValue(Map.class); @@ -94,7 +98,9 @@ public JSONObject enrich(CacheKey value) { } } } - return new JSONObject(message); + JSONObject enriched = new JSONObject(message); + _LOG.trace("Stellar Enrichment Success: " + enriched); + return enriched; } @Override diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java index cb7ebe0945..3f16b55d85 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java @@ -89,16 +89,17 @@ public JSONObject enrich(CacheKey value) { String enrichmentType = enrichmentTypes.get(i++); if (isThreat) { enriched.put(enrichmentType, "alert"); - _LOG.trace("Enriched value => " + enriched); + _LOG.trace("Theat Intel Enriched value => ", enriched); } } } catch(IOException e) { _LOG.error("Unable to retrieve value: " + e.getMessage(), e); initializeAdapter(); - throw new RuntimeException("Unable to retrieve value", e); + throw new RuntimeException("Theat Intel Unable to retrieve value", e); } } + _LOG.trace("Threat Intel Enrichment Success:", enriched); return enriched; } From f5da9ea59cc8473eff0ad6edd79653f37623fc90 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 10:24:07 -0400 Subject: [PATCH 65/82] Updating rhel ambari url --- metron-deployment/roles/ambari_common/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml index 8612cfad33..eda4a06417 100644 --- a/metron-deployment/roles/ambari_common/defaults/main.yml +++ b/metron-deployment/roles/ambari_common/defaults/main.yml @@ -18,4 +18,4 @@ hadoop_logrotate_frequency: daily hadoop_logrotate_retention: 30 -rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.2.2.0/ambari.repo" \ No newline at end of file +rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.0.1/ambari.repo" \ No newline at end of file From f474de9588db69579c5f07d85b6e5859b83a4c29 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 11:03:40 -0400 Subject: [PATCH 66/82] Upping NN heap to 2GB to handle history server --- .../roles/ambari_config/vars/multi_vagrant_cluster.yml | 2 +- metron-deployment/roles/ambari_config/vars/single_node_vm.yml | 2 +- metron-deployment/roles/ambari_config/vars/small_cluster.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml index 05291f1f8a..18d3b8d1f5 100644 --- a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml @@ -48,7 +48,7 @@ configurations: dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' - hadoop-env: - namenode_heapsize: 1024 + namenode_heapsize: 2048 dtnode_heapsize: 1024 - hbase-env: hbase_regionserver_heapsize: 1024 diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index 8b24735ec2..429981e765 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -39,7 +39,7 @@ configurations: dataDir: '{{ zookeeper_data_dir }}' - hadoop-env: hadoop_heapsize: 1024 - namenode_heapsize: 512 + namenode_heapsize: 2048 dtnode_heapsize: 512 namenode_opt_permsize: 128m - hbase-env: diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 72b2bbafac..a976d007dc 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -42,7 +42,7 @@ configurations: - zoo.cfg: dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' - hadoop-env: - namenode_heapsize: 1024 + namenode_heapsize: 2048 dtnode_heapsize: 1024 - hbase-env: hbase_regionserver_heapsize: 1024 From 594d9c0f68e5eebfead4e9db99800d6d6c438a1a Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 12:06:28 -0400 Subject: [PATCH 67/82] Fixing format() calls for Python 2.6 builtins --- .../package/scripts/enrichment_commands.py | 24 +++++++-------- .../package/scripts/indexing_commands.py | 24 +++++++-------- .../package/scripts/metron_service.py | 12 ++++---- .../package/scripts/parser_commands.py | 30 +++++++++---------- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 798da1c237..8c8b0d2b3c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -68,24 +68,24 @@ def remote_repo(): if repo_type in yum_repo_types: yum_repo_types[repo_type]() else: - raise ValueError("Unsupported repo type '{}'".format(repo_type)) + raise ValueError("Unsupported repo type '{0}'".format(repo_type)) def init_kafka_topics(self): Logger.info('Creating Kafka topics') - command_template = """{}/kafka-topics.sh \ - --zookeeper {} \ + command_template = """{0}/kafka-topics.sh \ + --zookeeper {1} \ --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""" + --topic {2} \ + --partitions {3} \ + --replication-factor {4} \ + --config retention.bytes={5}""" num_partitions = 1 replication_factor = 1 retention_gigabytes = 10 retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for enrichment") - Logger.info("Creating topic'{}'".format(self.__enrichment)) + Logger.info("Creating topic'{0}'".format(self.__enrichment)) Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, self.__enrichment, @@ -95,10 +95,10 @@ def init_kafka_topics(self): Logger.info("Done creating Kafka topics") def start_enrichment_topology(self): - Logger.info("Starting Metron enrichment topology: {}".format(self.__enrichment)) - start_cmd_template = """{}/bin/start_enrichment_topology.sh \ - -s {} \ - -z {}""" + Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment)) + start_cmd_template = """{0}/bin/start_enrichment_topology.sh \ + -s {1} \ + -z {2}""" Logger.info('Starting ' + self.__enrichment) Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment, self.__params.zookeeper_quorum)) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 95dc9baf1c..885331bb99 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -68,24 +68,24 @@ def remote_repo(): if repo_type in yum_repo_types: yum_repo_types[repo_type]() else: - raise ValueError("Unsupported repo type '{}'".format(repo_type)) + raise ValueError("Unsupported repo type '{0}'".format(repo_type)) def init_kafka_topics(self): Logger.info('Creating Kafka topics') - command_template = """{}/kafka-topics.sh \ - --zookeeper {} \ + command_template = """{0}/kafka-topics.sh \ + --zookeeper {1} \ --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""" + --topic {2} \ + --partitions {3} \ + --replication-factor {4} \ + --config retention.bytes={5}""" num_partitions = 1 replication_factor = 1 retention_gigabytes = int(self.__params.metron_topic_retention) retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for indexing") - Logger.info("Creating topic'{}'".format(self.__indexing)) + Logger.info("Creating topic'{0}'".format(self.__indexing)) Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, self.__indexing, @@ -95,10 +95,10 @@ def init_kafka_topics(self): Logger.info("Done creating Kafka topics") def start_indexing_topology(self): - Logger.info("Starting Metron indexing topology: {}".format(self.__indexing)) - start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \ - -s {} \ - -z {}""" + Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing)) + start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \ + -s {1} \ + -z {2}""" Logger.info('Starting ' + self.__indexing) Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum)) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py index 5fcc9c7540..34eb106bc8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -16,7 +16,7 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import format +from resource_management.libraries.functions import format as ambari_format from resource_management.core.resources.system import Directory, File from resource_management.core.source import InlineTemplate @@ -26,16 +26,16 @@ def init_config(): Logger.info('Loading config into ZooKeeper') - Execute(format( + Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), - path=format("{java_home}/bin") + path=ambari_format("{java_home}/bin") ) def get_running_topologies(): Logger.info('Getting Running Storm Topologies from Storm REST Server') - cmd = format('curl {storm_rest_addr}/api/v1/topology/summary') + cmd = ambari_format('curl {storm_rest_addr}/api/v1/topology/summary') proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() @@ -65,12 +65,12 @@ def load_global_config(params): group=params.metron_group ) - File("{}/global.json".format(params.metron_zookeeper_config_path), + File("{0}/global.json".format(params.metron_zookeeper_config_path), owner=params.metron_user, content=InlineTemplate(params.global_json_template) ) - File("{}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'), + File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'), owner=params.metron_user, content=InlineTemplate(params.global_properties_template)) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index 34b569dcf6..38f39571ce 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -25,8 +25,6 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File -from resource_management.core.source import Template -from resource_management.libraries.functions import format import metron_service # Wrap major operations and functionality in this class @@ -57,7 +55,7 @@ def set_configured(self): def init_parsers(self): Logger.info( - "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir, + "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir, self.__params.metron_apps_dir)) self.__params.HdfsResource(self.__params.metron_apps_dir, type="directory", @@ -94,24 +92,24 @@ def remote_repo(): if repo_type in yum_repo_types: yum_repo_types[repo_type]() else: - raise ValueError("Unsupported repo type '{}'".format(repo_type)) + raise ValueError("Unsupported repo type '{0}'".format(repo_type)) def init_kafka_topics(self): Logger.info('Creating Kafka topics') - command_template = """{}/kafka-topics.sh \ - --zookeeper {} \ + command_template = """{0}/kafka-topics.sh \ + --zookeeper {1} \ --create \ - --topic {} \ - --partitions {} \ - --replication-factor {} \ - --config retention.bytes={}""" + --topic {2} \ + --partitions {3} \ + --replication-factor {4} \ + --config retention.bytes={5}""" num_partitions = 1 replication_factor = 1 retention_gigabytes = 10 retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): - Logger.info("Creating topic'{}'".format(parser_name)) + Logger.info("Creating topic'{0}'".format(parser_name)) Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, parser_name, @@ -133,11 +131,11 @@ def init_kafka_topics(self): Logger.info("Done creating Kafka topics") def start_parser_topologies(self): - Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list())) - start_cmd_template = """{}/bin/start_parser_topology.sh \ - -k {} \ - -z {} \ - -s {}""" + Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list())) + start_cmd_template = """{1}/bin/start_parser_topology.sh \ + -k {2} \ + -z {3} \ + -s {4}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, From ba8aae96fde775f1553ea81f5a237e554ccc1fce Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 13:12:35 -0400 Subject: [PATCH 68/82] Python 2.6 compatability fix in parser_commands --- .../METRON/0.2.0BETA/package/scripts/parser_commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index 38f39571ce..d59fa6d685 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -132,10 +132,10 @@ def init_kafka_topics(self): def start_parser_topologies(self): Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list())) - start_cmd_template = """{1}/bin/start_parser_topology.sh \ - -k {2} \ - -z {3} \ - -s {4}""" + start_cmd_template = """{0}/bin/start_parser_topology.sh \ + -k {1} \ + -z {2} \ + -s {3}""" for parser in self.get_parser_list(): Logger.info('Starting ' + parser) Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, From 09ee8061d232d69c8d3a5fee37c1c757adc34735 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 15:28:23 -0400 Subject: [PATCH 69/82] Fixing gitignore --- metron-deployment/packaging/ambari/.gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore index 1b65a68dc5..e708548071 100644 --- a/metron-deployment/packaging/ambari/.gitignore +++ b/metron-deployment/packaging/ambari/.gitignore @@ -1,6 +1,2 @@ archive.zip *.hash -<<<<<<< HEAD - -======= ->>>>>>> master From f013c2980e81be0a80891b78bc2f1b410afa9b4d Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 14 Sep 2016 16:01:23 -0400 Subject: [PATCH 70/82] Removing the non-mpack wayservices --- .../2.3.3/configuration/elastic-env.xml | 58 - .../2.3.3/configuration/elastic-site.xml | 180 -- .../2.3.3/configuration/elastic-sysconfig.xml | 98 -- .../ELASTICSEARCH/2.3.3/metainfo.xml | 73 - .../2.3.3/package/scripts/elastic.py | 68 - .../2.3.3/package/scripts/elastic_master.py | 79 - .../2.3.3/package/scripts/elastic_slave.py | 76 - .../2.3.3/package/scripts/params.py | 77 - .../package/scripts/properties_config.py | 34 - .../2.3.3/package/scripts/service_check.py | 80 - .../2.3.3/package/scripts/slave.py | 65 - .../2.3.3/package/scripts/status_params.py | 26 - .../templates/elasticsearch.master.yaml.j2 | 84 - .../templates/elasticsearch.slave.yaml.j2 | 84 - .../2.3.3/role_command_order.json | 8 - .../KIBANA/4.5.1/configuration/kibana-env.xml | 54 - .../4.5.1/configuration/kibana-site.xml | 112 -- .../common-services/KIBANA/4.5.1/metainfo.xml | 75 - .../package/scripts/dashboard/__init__.py | 16 - .../package/scripts/dashboard/dashboard.p | 1539 ----------------- .../scripts/dashboard/dashboardindex.py | 95 - .../4.5.1/package/scripts/kibana_master.py | 146 -- .../KIBANA/4.5.1/package/scripts/params.py | 47 - .../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 - .../2.4/services/ELASTICSEARCH/metainfo.xml | 41 - .../HDP/2.4/services/KIBANA/metainfo.xml | 29 - 26 files changed, 3272 deletions(-) delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml delete mode 100755 metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml deleted file mode 100755 index 2939c284a0..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - elastic_user - elasticsearch - USER - The user for Elasticsearch - - - user_group - elasticsearch - The group for Elasticsearch - - - elastic_log_dir - /var/log/elasticsearch - Log directory for elastic - - - elastic_pid_dir - /var/run/elasticsearch - The directory for pid files - - - - - content - This is the jinja template for elastic-env.sh file - -#!/bin/bash - -# Set ELASTICSEARCH specific environment variables here. - -# The java implementation to use. -export JAVA_HOME={{java64_home}} -export PATH=$PATH:$JAVA_HOME/bin - - - diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml deleted file mode 100755 index fb3a44395d..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ /dev/null @@ -1,180 +0,0 @@ - - - - - - - - - cluster_name - metron - Cluster name identifies your cluster - - - zen_discovery_ping_unicast_hosts - - - Unicast discovery list of hosts to act as gossip routers, in comma separated format. - - - index_number_of_shards - 4 - Set the number of shards (splits) of an index - - - index_number_of_replicas - 2 - Set the number of replicas (additional copies) of an index - - - - path_data - "/opt/lmm/es_data" - Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second" - - - - transport_tcp_port - 9300-9400 - Set a custom port for the node to node communication - - - http_port - 9200-9300 - Set a custom port to listen for HTTP traffic - - - - discovery_zen_ping_multicast_enabled - false - master eligible nodes - - - discovery_zen_ping_timeout - 3s - Wait for ping responses for master discovery - - - discovery_zen_fd_ping_interval - 15s - Wait for ping for cluster discovery - - - discovery_zen_fd_ping_timeout - 60s - Wait for ping for cluster discovery - - - discovery_zen_fd_ping_retries - 5 - Number of ping retries before blacklisting - - - - gateway_recover_after_data_nodes - 3 - Recover as long as this many data or master nodes have joined the cluster. - - - recover_after_time - 15m - recover_after_time - - - expected_data_nodes - 0 - expected_data_nodes - - - - index_merge_scheduler_max_thread_count - 5 - index.merge.scheduler.max_thread_count - - - indices_memory_index_store_throttle_type - none - index_store_throttle_type - - - index_refresh_interval - 1s - index refresh interval - - - index_translog_flush_threshold_size - 5g - index_translog_flush_threshold_size - - - indices_memory_index_buffer_size - 10% - Percentage of heap used for write buffers - - - bootstrap_mlockall - true - The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out - - - threadpool_bulk_queue_size - 3000 - It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request - - - threadpool_index_queue_size - 1000 - It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request - - - indices_cluster_send_refresh_mapping - false - In order to make the index request more efficient, we have set this property on our data nodes - - - indices_fielddata_cache_size - 25% - You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index - - - cluster_routing_allocation_disk_watermark_high - 0.99 - Property used when multiple drives are used to understand max thresholds - - - cluster_routing_allocation_disk_threshold_enabled - true - Property used when multiple drives are used to understand if thresholding is active - - - cluster_routing_allocation_disk_watermark_low - .97 - Property used when multiple drives are used to understand min thresholds - - - cluster_routing_allocation_node_concurrent_recoveries - 4 - Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart - - - network_host - _lo_,_eth0_ - Network interface(s) will bind to. - - \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml deleted file mode 100755 index 58e4916062..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - - - - elastic_home - /usr/share/elasticsearch/ - Elasticsearch Home Directory - - - data_dir - /var/lib/elasticsearch/ - Elasticsearch Data Directory - - - work_dir - /tmp/elasticsearch/ - Elasticsearch Work Directory - - - conf_dir - /etc/elasticsearch/ - Elasticsearch Configuration Directory - - - heap_size - 128m - Heap size - - - max_open_files - 65535 - Maximum number of open files - - - max_map_count - 262144 - Maximum number of memory map areas for process - - - - - content - This is the jinja template for elastic-env.sh file - -# Directory where the Elasticsearch binary distribution resides -ES_HOME={{elastic_home}} - -# Heap Size (defaults to 256m min, 1g max) -ES_HEAP_SIZE={{heap_size}} - -# Maximum number of open files -MAX_OPEN_FILES={{max_open_files}} - -# Maximum number of VMA (Virtual Memory Areas) a process can own -MAX_MAP_COUNT={{max_map_count}} - -# Elasticsearch log directory -LOG_DIR={{log_dir}} - -# Elasticsearch data directory -DATA_DIR={{data_dir}} - -# Elasticsearch work directory -WORK_DIR={{work_dir}} - -# Elasticsearch conf directory -CONF_DIR={{conf_dir}} - -# User to run as, change this to a specific elasticsearch user if possible -# Also make sure, this user can write into the log directories in case you change them -# This setting only works for the init script, but has to be configured separately for systemd startup -ES_USER={{elastic_user}} - -# Additional Java OPTS -ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled --XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps --XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8" - - - diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml deleted file mode 100755 index a420131564..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - 2.0 - - - ELASTICSEARCH - Elasticsearch - Indexing and Search - 2.3.3 - - - ES_MASTER - Elasticsearch Master-Eligible Node - MASTER - 1+ - - - PYTHON - 600 - - - - ES_SLAVE - Elasticsearch Data Node - SLAVE - 0+ - - - PYTHON - 600 - - - - - - any - - - elasticsearch-2.3.3 - - - - - - - PYTHON - 300 - - - elastic-env - elastic-site - elastic-sysconfig - - true - - - \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py deleted file mode 100755 index bd858cd66d..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from resource_management.core.source import Template - - -def elastic(): - print "INSIDE THE %s" % __file__ - import params - - params.path_data = params.path_data.replace('"', '') - data_path = params.path_data.replace(' ', '').split(',') - data_path[:] = [x.replace('"', '') for x in data_path] - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - directories = directories + data_path - - Directory(directories, - create_parents=True, - # recursive=True, - mode=0755, - owner=params.elastic_user, - group=params.elastic_user - ) - - print "Master env: ""{}/elastic-env.sh".format(params.conf_dir) - File("{}/elastic-env.sh".format(params.conf_dir), - owner=params.elastic_user, - content=InlineTemplate(params.elastic_env_sh_template) - ) - - configurations = params.config['configurations']['elastic-site'] - - print "Master yml: ""{}/elasticsearch.yml".format(params.conf_dir) - File("{}/elasticsearch.yml".format(params.conf_dir), - content=Template( - "elasticsearch.master.yaml.j2", - configurations=configurations), - owner=params.elastic_user, - group=params.elastic_user - ) - - print "Master sysconfig: /etc/sysconfig/elasticsearch" - File(format("/etc/sysconfig/elasticsearch"), - owner="root", - group="root", - content=InlineTemplate(params.sysconfig_template) - ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py deleted file mode 100755 index 5fc29cf32b..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from elastic import elastic -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script - - -class Elasticsearch(Script): - def install(self, env): - import params - env.set_params(params) - - print 'Install the Master' - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Execute("echo \"[elasticsearch-2.x]\n" - "name=Elasticsearch repository for 2.x packages\n" - "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") - - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - - elastic() - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - stop_cmd = format("service elasticsearch stop") - print 'Stop the Master' - Execute(stop_cmd) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - - self.configure(env) - start_cmd = format("service elasticsearch start") - print 'Start the Master' - Execute(start_cmd) - - def status(self, env): - import params - env.set_params(params) - status_cmd = format("service elasticsearch status") - print 'Status of the Master' - Execute(status_cmd) - - def restart(self, env): - import params - env.set_params(params) - self.configure(env) - restart_cmd = format("service elasticsearch restart") - print 'Restarting the Master' - Execute(restart_cmd) - -if __name__ == "__main__": - Elasticsearch().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py deleted file mode 100755 index e65bd8f28e..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script -from resource_management.core.resources.system import Execute -from slave import slave - - -class Elasticsearch(Script): - def install(self, env): - import params - env.set_params(params) - print 'Install the Slave' - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Execute("echo \"[elasticsearch-2.x]\n" - "name=Elasticsearch repository for 2.x packages\n" - "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - slave() - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - stop_cmd = format("service elasticsearch stop") - print 'Stop the Slave' - Execute(stop_cmd) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - start_cmd = format("service elasticsearch start") - print 'Start the Slave' - Execute(start_cmd) - - def status(self, env): - import params - env.set_params(params) - status_cmd = format("service elasticsearch status") - print 'Status of the Slave' - Execute(status_cmd) - - def restart(self, env): - import params - env.set_params(params) - self.configure(env) - restart_cmd = format("service elasticsearch restart") - print 'Restarting the Slave' - Execute(restart_cmd) - - -if __name__ == "__main__": - Elasticsearch().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py deleted file mode 100755 index c3e9169416..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -# server configurations -config = Script.get_config() - -elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] -data_dir = config['configurations']['elastic-sysconfig']['data_dir'] -work_dir = config['configurations']['elastic-sysconfig']['work_dir'] -conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] -heap_size = config['configurations']['elastic-sysconfig']['heap_size'] -max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] -max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] - -elastic_user = config['configurations']['elastic-env']['elastic_user'] -user_group = config['configurations']['elastic-env']['user_group'] -log_dir = config['configurations']['elastic-env']['elastic_log_dir'] -pid_dir = '/var/run/elasticsearch' -pid_file = '/var/run/elasticsearch/elasticsearch.pid' -hostname = config['hostname'] -java64_home = config['hostLevelParams']['java_home'] -elastic_env_sh_template = config['configurations']['elastic-env']['content'] -sysconfig_template = config['configurations']['elastic-sysconfig']['content'] - -cluster_name = config['configurations']['elastic-site']['cluster_name'] -zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] - -path_data = config['configurations']['elastic-site']['path_data'] -http_port = config['configurations']['elastic-site']['http_port'] -transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] - -recover_after_time = config['configurations']['elastic-site']['recover_after_time'] -gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] -expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] -discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled'] -index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] -index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] -index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] -indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] -index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] -index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] -indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] -bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall'] -threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] -cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] -cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] -cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'] -cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] -indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] -indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'] -threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] - -discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] -discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval'] -discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] -discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] - -network_host = config['configurations']['elastic-site']['network_host'] \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py deleted file mode 100755 index ef9f6dd77d..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate - - -def properties_inline_template(configurations): - return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }} -{% endfor %}''', configurations_dict=configurations) - - -def properties_config(filename, configurations=None, conf_dir=None, - mode=None, owner=None, group=None, brokerid=None): - config_content = properties_inline_template(configurations) - File(format("{conf_dir}/{filename}"), content=config_content, owner=owner, - group=group, mode=mode) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py deleted file mode 100755 index 9615d83405..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from __future__ import print_function - -import sys - -from resource_management.libraries.script import Script -from resource_management.core.resources.system import Execute -import subprocess - - -class ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - doc = '{"name": "Ambari Smoke test"}' - index = "ambari_smoke_test" - - print("Running Elastic search service check", file=sys.stdout) - - # Make sure the service is actually up. We can live without everything allocated. - # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can - # also hit ES before things are green. - host = "localhost:9200" - Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host, - logoutput=True, - tries=6, - try_sleep=20 - ) - - # Put a document into a new index. - - Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) - - # Retrieve the document. Use subprocess because we actually need the results here. - cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index) - proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - response_retrieve = stdout - print("Retrieval response is: %s" % response_retrieve) - expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ - % (index, doc) - - # Delete the index - cmd_delete = "curl -XDELETE '%s/%s'" % (host, index) - proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - response_delete = stdout - print("Delete index response is: %s" % response_retrieve) - expected_delete = '{"acknowledged":true}' - - if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): - print("Smoke test able to communicate with Elasticsearch") - else: - print("Elasticsearch service unable to retrieve document.") - sys.exit(1) - - exit(0) - - -if __name__ == "__main__": - ServiceCheck().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py deleted file mode 100755 index a134160e13..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from resource_management.core.source import Template - - -def slave(): - import params - - params.path_data = params.path_data.replace('"', '') - data_path = params.path_data.replace(' ', '').split(',') - data_path[:] = [x.replace('"', '') for x in data_path] - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - directories = directories + data_path - - Directory(directories, - create_parents=True, - mode=0755, - owner=params.elastic_user, - group=params.elastic_user, - cd_access="a" - ) - - File("{}/elastic-env.sh".format(params.conf_dir), - owner=params.elastic_user, - content=InlineTemplate(params.elastic_env_sh_template) - ) - - configurations = params.config['configurations']['elastic-site'] - - File("{}/elasticsearch.yml".format(params.conf_dir), - content=Template( - "elasticsearch.slave.yaml.j2", - configurations=configurations), - owner=params.elastic_user, - group=params.elastic_user - ) - - print "Master sysconfig: /etc/sysconfig/elasticsearch" - File(format("/etc/sysconfig/elasticsearch"), - owner="root", - group="root", - content=InlineTemplate(params.sysconfig_template) - ) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py deleted file mode 100755 index 9cfb5cf014..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -config = Script.get_config() - -elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] -elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid") diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 deleted file mode 100755 index a9de018450..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 +++ /dev/null @@ -1,84 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -cluster: - name: {{cluster_name}} - routing: - allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} - allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} - allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} - allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} - -discovery: - zen: - ping: - multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} - unicast: - hosts: "{{zen_discovery_ping_unicast_hosts}}" - -node: - data: false - master: true - name: {{hostname}} -path: - data: {{path_data}} - -http.cors.enabled: true - -port: {{http_port}} - -transport: - tcp: - port: {{transport_tcp_port}} - -gateway: - recover_after_data_nodes: {{gateway_recover_after_data_nodes}} - recover_after_time: {{recover_after_time}} - expected_data_nodes: {{expected_data_nodes}} - -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - -indices: - memory: - index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} - fielddata: - cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} - -bootstrap.mlockall: {{bootstrap_mlockall}} - -threadpool: - bulk: - queue_size: {{threadpool_bulk_queue_size}} - index: - queue_size: {{threadpool_index_queue_size}} - -discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} -discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} -discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} -discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} - -network.host: {{network_host}} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 deleted file mode 100755 index e88fc5f0b0..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 +++ /dev/null @@ -1,84 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -cluster: - name: {{cluster_name}} - routing: - allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} - allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} - allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} - allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} - -discovery: - zen: - ping: - multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} - unicast: - hosts: "{{zen_discovery_ping_unicast_hosts}}" - -node: - data: true - master: false - name: {{hostname}} -path: - data: {{path_data}} - -http.cors.enabled: true - -port: {{http_port}} - -transport: - tcp: - port: {{transport_tcp_port}} - -gateway: - recover_after_data_nodes: {{gateway_recover_after_data_nodes}} - recover_after_time: {{recover_after_time}} - expected_data_nodes: {{expected_data_nodes}} - -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - -indices: - memory: - index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} - fielddata: - cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} - -bootstrap.mlockall: {{bootstrap_mlockall}} - -threadpool: - bulk: - queue_size: {{threadpool_bulk_queue_size}} - index: - queue_size: {{threadpool_index_queue_size}} - -discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} -discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} -discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} -discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} - -network.host: {{network_host}} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json deleted file mode 100755 index 130d018de6..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"] - } -} diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml deleted file mode 100755 index 972945dfba..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - kibana_user - kibana - USER - - - - user_group - kibana - GROUP - - - - kibana_log_dir - /var/log/kibana - Log directory for Kibana - - - kibana_pid_dir - /var/run/kibana - PID directory for Kibana - - - kibana_es_url - - The Elasticsearch instance to use for all your queries. (http://eshost:9200) - - - kibana_server_port - 5000 - Kibana back end server port to use. - - diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml deleted file mode 100755 index c546e2c2ba..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - - content - kibana.yml template - This is the jinja template for kibana.yml file - -# Kibana is served by a back end server. This controls which port to use. -server.port: {{ kibana_port }} - -# The host to bind the server to. -# server.host: "0.0.0.0" - -# If you are running kibana behind a proxy, and want to mount it at a path, -# specify that path here. The basePath can't end in a slash. -# server.basePath: "" - -# The maximum payload size in bytes on incoming server requests. -# server.maxPayloadBytes: 1048576 - -# The Elasticsearch instance to use for all your queries. -elasticsearch.url: {{ es_url }} - -# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, -# then the host you use to connect to *this* Kibana instance will be sent. -# elasticsearch.preserveHost: true - -# Kibana uses an index in Elasticsearch to store saved searches, visualizations -# and dashboards. It will create a new index if it doesn't already exist. -# kibana.index: ".kibana" - -# The default application to load. -# kibana.defaultAppId: "discover" - -# If your Elasticsearch is protected with basic auth, these are the user credentials -# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana -# users will still need to authenticate with Elasticsearch (which is proxied through -# the Kibana server) -# elasticsearch.username: "user" -# elasticsearch.password: "pass" - -# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) -# server.ssl.cert: /path/to/your/server.crt -# server.ssl.key: /path/to/your/server.key - -# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) -# elasticsearch.ssl.cert: /path/to/your/client.crt -# elasticsearch.ssl.key: /path/to/your/client.key - -# If you need to provide a CA certificate for your Elasticsearch instance, put -# the path of the pem file here. -# elasticsearch.ssl.ca: /path/to/your/CA.pem - -# Set to false to have a complete disregard for the validity of the SSL -# certificate. -# elasticsearch.ssl.verify: true - -# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to -# request_timeout setting -# elasticsearch.pingTimeout: 1500 - -# Time in milliseconds to wait for responses from the back end or elasticsearch. -# This must be > 0 -# elasticsearch.requestTimeout: 30000 - -# Time in milliseconds for Elasticsearch to wait for responses from shards. -# Set to 0 to disable. -# elasticsearch.shardTimeout: 0 - -# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying -# elasticsearch.startupTimeout: 5000 - -# Set the path to where you would like the process id file to be created. -# pid.file: /var/run/kibana.pid - -# If you would like to send the log output to a file you can set the path below. -logging.dest: {{ log_dir }}/kibana.log - -# Set this to true to suppress all logging output. -# logging.silent: false - -# Set this to true to suppress all logging output except for error messages. -# logging.quiet: false - -# Set this to true to log all events, including system usage information and all requests. -# logging.verbose: false - - - content - - - diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml deleted file mode 100755 index d14afbf5e8..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 2.0 - - - KIBANA - Kibana - Kibana Dashboard - 4.5.1 - - - KIBANA_MASTER - Kibana Server - MASTER - 1 - - - PYTHON - 600 - - - - LOADTEMPLATE - false - - - PYTHON - - - - - - - - any - - - python-elasticsearch - - - kibana-4.5.1 - - - - - - kibana-env - kibana-site - - true - - - quicklinks.json - true - - - - - diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py deleted file mode 100755 index 8d2bad81df..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p deleted file mode 100755 index 8327eb8ad9..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p +++ /dev/null @@ -1,1539 +0,0 @@ -(lp1 -(dp2 -V_score -p3 -F1 -sV_type -p4 -Vvisualization -p5 -sV_id -p6 -VWeb-Request-Type -p7 -sV_source -p8 -(dp9 -VvisState -p10 -V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p11 -sVdescription -p12 -V -sVtitle -p13 -VWeb Request Type -p14 -sVuiStateJSON -p15 -V{} -p16 -sVversion -p17 -I1 -sVsavedSearchId -p18 -Vweb-search -p19 -sVkibanaSavedObjectMeta -p20 -(dp21 -VsearchSourceJSON -p22 -V{"filter":[]} -p23 -sssV_index -p24 -V.kibana -p25 -sa(dp26 -V_score -p27 -F1 -sV_type -p28 -Vvisualization -p29 -sV_id -p30 -VTop-Snort-Alerts-by-Source -p31 -sV_source -p32 -(dp33 -VvisState -p34 -V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} -p35 -sVdescription -p36 -V -sVtitle -p37 -VTop Snort Alerts by Source -p38 -sVuiStateJSON -p39 -V{} -p40 -sVversion -p41 -I1 -sVkibanaSavedObjectMeta -p42 -(dp43 -VsearchSourceJSON -p44 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p45 -sssV_index -p46 -V.kibana -p47 -sa(dp48 -V_score -p49 -F1 -sV_type -p50 -Vvisualization -p51 -sV_id -p52 -VWelcome -p53 -sV_source -p54 -(dp55 -VvisState -p56 -V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} -p57 -sVdescription -p58 -V -sVtitle -p59 -VWelcome to Apache Metron -p60 -sVuiStateJSON -p61 -V{} -p62 -sVversion -p63 -I1 -sVkibanaSavedObjectMeta -p64 -(dp65 -VsearchSourceJSON -p66 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p67 -sssV_index -p68 -V.kibana -p69 -sa(dp70 -V_score -p71 -F1 -sV_type -p72 -Vsearch -p73 -sV_id -p74 -Vsnort-search -p75 -sV_source -p76 -(dp77 -Vsort -p78 -(lp79 -Vtimestamp -p80 -aVdesc -p81 -asVhits -p82 -I0 -sVdescription -p83 -V -sVtitle -p84 -VSnort Alerts -p85 -sVversion -p86 -I1 -sVkibanaSavedObjectMeta -p87 -(dp88 -VsearchSourceJSON -p89 -V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p90 -ssVcolumns -p91 -(lp92 -Vmsg -p93 -aVsig_id -p94 -aVip_src_addr -p95 -aVip_src_port -p96 -aVip_dst_addr -p97 -aVip_dst_port -p98 -assV_index -p99 -V.kibana -p100 -sa(dp101 -V_score -p102 -F1 -sV_type -p103 -Vsearch -p104 -sV_id -p105 -Vyaf-search -p106 -sV_source -p107 -(dp108 -Vsort -p109 -(lp110 -Vtimestamp -p111 -aVdesc -p112 -asVhits -p113 -I0 -sVdescription -p114 -V -sVtitle -p115 -VYAF -p116 -sVversion -p117 -I1 -sVkibanaSavedObjectMeta -p118 -(dp119 -VsearchSourceJSON -p120 -V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} -p121 -ssVcolumns -p122 -(lp123 -Vip_src_addr -p124 -aVip_src_port -p125 -aVip_dst_addr -p126 -aVip_dst_port -p127 -aVprotocol -p128 -aVduration -p129 -aVpkt -p130 -assV_index -p131 -V.kibana -p132 -sa(dp133 -V_score -p134 -F1 -sV_type -p135 -Vconfig -p136 -sV_id -p137 -V4.5.1 -p138 -sV_source -p139 -(dp140 -VbuildNum -p141 -I9892 -sVdefaultIndex -p142 -Vbro* -p143 -ssV_index -p144 -V.kibana -p145 -sa(dp146 -V_score -p147 -F1 -sV_type -p148 -Vindex-pattern -p149 -sV_id -p150 -Vbro* -p151 -sV_source -p152 -(dp153 -Vfields -p154 -V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p155 -sVtimeFieldName -p156 -Vtimestamp -p157 -sVtitle -p158 -Vbro* -p159 -ssV_index -p160 -V.kibana -p161 -sa(dp162 -V_score -p163 -F1 -sV_type -p164 -Vvisualization -p165 -sV_id -p166 -VFlow-Duration -p167 -sV_source -p168 -(dp169 -VvisState -p170 -V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} -p171 -sVdescription -p172 -V -sVtitle -p173 -VFlow Duration -p174 -sVuiStateJSON -p175 -V{"vis":{"legendOpen":false}} -p176 -sVversion -p177 -I1 -sVkibanaSavedObjectMeta -p178 -(dp179 -VsearchSourceJSON -p180 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p181 -sssV_index -p182 -V.kibana -p183 -sa(dp184 -V_score -p185 -F1 -sV_type -p186 -Vvisualization -p187 -sV_id -p188 -VEvents -p189 -sV_source -p190 -(dp191 -VvisState -p192 -V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p193 -sVdescription -p194 -V -sVtitle -p195 -VEvents -p196 -sVuiStateJSON -p197 -V{"vis":{"legendOpen":false}} -p198 -sVversion -p199 -I1 -sVkibanaSavedObjectMeta -p200 -(dp201 -VsearchSourceJSON -p202 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p203 -sssV_index -p204 -V.kibana -p205 -sa(dp206 -V_score -p207 -F1 -sV_type -p208 -Vvisualization -p209 -sV_id -p210 -VWeb-Request-Header -p211 -sV_source -p212 -(dp213 -VvisState -p214 -V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} -p215 -sVdescription -p216 -V -sVtitle -p217 -VWeb Request Header -p218 -sVuiStateJSON -p219 -V{} -p220 -sVversion -p221 -I1 -sVkibanaSavedObjectMeta -p222 -(dp223 -VsearchSourceJSON -p224 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p225 -sssV_index -p226 -V.kibana -p227 -sa(dp228 -V_score -p229 -F1 -sV_type -p230 -Vvisualization -p231 -sV_id -p232 -VTop-Alerts-By-Host -p233 -sV_source -p234 -(dp235 -VvisState -p236 -V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} -p237 -sVdescription -p238 -V -sVtitle -p239 -VTop Alerts By Host -p240 -sVuiStateJSON -p241 -V{} -p242 -sVversion -p243 -I1 -sVsavedSearchId -p244 -Vsnort-search -p245 -sVkibanaSavedObjectMeta -p246 -(dp247 -VsearchSourceJSON -p248 -V{"filter":[]} -p249 -sssV_index -p250 -V.kibana -p251 -sa(dp252 -V_score -p253 -F1 -sV_type -p254 -Vvisualization -p255 -sV_id -p256 -VYAF-Flow(s) -p257 -sV_source -p258 -(dp259 -VvisState -p260 -V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p261 -sVdescription -p262 -V -sVtitle -p263 -VYAF Flows -p264 -sVuiStateJSON -p265 -V{} -p266 -sVversion -p267 -I1 -sVkibanaSavedObjectMeta -p268 -(dp269 -VsearchSourceJSON -p270 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p271 -sssV_index -p272 -V.kibana -p273 -sa(dp274 -V_score -p275 -F1 -sV_type -p276 -Vvisualization -p277 -sV_id -p278 -VTop-DNS-Query -p279 -sV_source -p280 -(dp281 -VvisState -p282 -V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p283 -sVdescription -p284 -V -sVtitle -p285 -VTop DNS Query -p286 -sVuiStateJSON -p287 -V{} -p288 -sVversion -p289 -I1 -sVkibanaSavedObjectMeta -p290 -(dp291 -VsearchSourceJSON -p292 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p293 -sssV_index -p294 -V.kibana -p295 -sa(dp296 -V_score -p297 -F1 -sV_type -p298 -Vvisualization -p299 -sV_id -p300 -VTotal-Events -p301 -sV_source -p302 -(dp303 -VvisState -p304 -V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} -p305 -sVdescription -p306 -V -sVtitle -p307 -VEvent Count -p308 -sVuiStateJSON -p309 -V{} -p310 -sVversion -p311 -I1 -sVkibanaSavedObjectMeta -p312 -(dp313 -VsearchSourceJSON -p314 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p315 -sssV_index -p316 -V.kibana -p317 -sa(dp318 -V_score -p319 -F1 -sV_type -p320 -Vvisualization -p321 -sV_id -p322 -VEvent-Types -p323 -sV_source -p324 -(dp325 -VvisState -p326 -V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p327 -sVdescription -p328 -V -sVtitle -p329 -VEvent Sources -p330 -sVuiStateJSON -p331 -V{} -p332 -sVversion -p333 -I1 -sVkibanaSavedObjectMeta -p334 -(dp335 -VsearchSourceJSON -p336 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p337 -sssV_index -p338 -V.kibana -p339 -sa(dp340 -V_score -p341 -F1 -sV_type -p342 -Vvisualization -p343 -sV_id -p344 -VUnique-Location(s) -p345 -sV_source -p346 -(dp347 -VvisState -p348 -V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} -p349 -sVdescription -p350 -V -sVtitle -p351 -VGeo-IP Locations -p352 -sVuiStateJSON -p353 -V{} -p354 -sVversion -p355 -I1 -sVkibanaSavedObjectMeta -p356 -(dp357 -VsearchSourceJSON -p358 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p359 -sssV_index -p360 -V.kibana -p361 -sa(dp362 -V_score -p363 -F1 -sV_type -p364 -Vvisualization -p365 -sV_id -p366 -VSnort-Header -p367 -sV_source -p368 -(dp369 -VvisState -p370 -V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} -p371 -sVdescription -p372 -V -sVtitle -p373 -VSnort -p374 -sVuiStateJSON -p375 -V{} -p376 -sVversion -p377 -I1 -sVkibanaSavedObjectMeta -p378 -(dp379 -VsearchSourceJSON -p380 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p381 -sssV_index -p382 -V.kibana -p383 -sa(dp384 -V_score -p385 -F1 -sV_type -p386 -Vdashboard -p387 -sV_id -p388 -VMetron-Dashboard -p389 -sV_source -p390 -(dp391 -Vhits -p392 -I0 -sVtimeRestore -p393 -I00 -sVdescription -p394 -V -sVtitle -p395 -VMetron Dashboard -p396 -sVuiStateJSON -p397 -V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} -p398 -sVpanelsJSON -p399 -V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] -p400 -sVoptionsJSON -p401 -V{"darkTheme":false} -p402 -sVversion -p403 -I1 -sVkibanaSavedObjectMeta -p404 -(dp405 -VsearchSourceJSON -p406 -V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} -p407 -sssV_index -p408 -V.kibana -p409 -sa(dp410 -V_score -p411 -F1 -sV_type -p412 -Vvisualization -p413 -sV_id -p414 -VSnort-Alert-Types -p415 -sV_source -p416 -(dp417 -VvisState -p418 -V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} -p419 -sVdescription -p420 -V -sVtitle -p421 -VSnort Alert Types -p422 -sVuiStateJSON -p423 -V{} -p424 -sVversion -p425 -I1 -sVkibanaSavedObjectMeta -p426 -(dp427 -VsearchSourceJSON -p428 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p429 -sssV_index -p430 -V.kibana -p431 -sa(dp432 -V_score -p433 -F1 -sV_type -p434 -Vvisualization -p435 -sV_id -p436 -VFrequent-DNS-Queries -p437 -sV_source -p438 -(dp439 -VvisState -p440 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p441 -sVdescription -p442 -V -sVtitle -p443 -VFrequent DNS Requests -p444 -sVuiStateJSON -p445 -V{} -p446 -sVversion -p447 -I1 -sVkibanaSavedObjectMeta -p448 -(dp449 -VsearchSourceJSON -p450 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p451 -sssV_index -p452 -V.kibana -p453 -sa(dp454 -V_score -p455 -F1 -sV_type -p456 -Vvisualization -p457 -sV_id -p458 -VLocation-Header -p459 -sV_source -p460 -(dp461 -VvisState -p462 -V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} -p463 -sVdescription -p464 -V -sVtitle -p465 -VEnrichment -p466 -sVuiStateJSON -p467 -V{} -p468 -sVversion -p469 -I1 -sVkibanaSavedObjectMeta -p470 -(dp471 -VsearchSourceJSON -p472 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p473 -sssV_index -p474 -V.kibana -p475 -sa(dp476 -V_score -p477 -F1 -sV_type -p478 -Vsearch -p479 -sV_id -p480 -Vweb-search -p481 -sV_source -p482 -(dp483 -Vsort -p484 -(lp485 -Vtimestamp -p486 -aVdesc -p487 -asVhits -p488 -I0 -sVdescription -p489 -V -sVtitle -p490 -VWeb Requests -p491 -sVversion -p492 -I1 -sVkibanaSavedObjectMeta -p493 -(dp494 -VsearchSourceJSON -p495 -V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p496 -ssVcolumns -p497 -(lp498 -Vmethod -p499 -aVhost -p500 -aVuri -p501 -aVreferrer -p502 -aVip_src_addr -p503 -aVip_dst_addr -p504 -assV_index -p505 -V.kibana -p506 -sa(dp507 -V_score -p508 -F1 -sV_type -p509 -Vindex-pattern -p510 -sV_id -p511 -Vsnort* -p512 -sV_source -p513 -(dp514 -Vfields -p515 -V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p516 -sVtimeFieldName -p517 -Vtimestamp -p518 -sVtitle -p519 -Vsnort* -p520 -ssV_index -p521 -V.kibana -p522 -sa(dp523 -V_score -p524 -F1 -sV_type -p525 -Vindex-pattern -p526 -sV_id -p527 -Vyaf* -p528 -sV_source -p529 -(dp530 -Vfields -p531 -V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p532 -sVtimeFieldName -p533 -Vtimestamp -p534 -sVtitle -p535 -Vyaf* -p536 -ssV_index -p537 -V.kibana -p538 -sa(dp539 -V_score -p540 -F1 -sV_type -p541 -Vvisualization -p542 -sV_id -p543 -VDNS-Request(s) -p544 -sV_source -p545 -(dp546 -VvisState -p547 -V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p548 -sVdescription -p549 -V -sVtitle -p550 -VDNS Requests -p551 -sVuiStateJSON -p552 -V{} -p553 -sVversion -p554 -I1 -sVsavedSearchId -p555 -Vdns-search -p556 -sVkibanaSavedObjectMeta -p557 -(dp558 -VsearchSourceJSON -p559 -V{"filter":[]} -p560 -sssV_index -p561 -V.kibana -p562 -sa(dp563 -V_score -p564 -F1 -sV_type -p565 -Vvisualization -p566 -sV_id -p567 -VHTTP(S)-Requests -p568 -sV_source -p569 -(dp570 -VvisState -p571 -V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p572 -sVdescription -p573 -V -sVtitle -p574 -VWeb Requests -p575 -sVuiStateJSON -p576 -V{} -p577 -sVversion -p578 -I1 -sVsavedSearchId -p579 -Vweb-search -p580 -sVkibanaSavedObjectMeta -p581 -(dp582 -VsearchSourceJSON -p583 -V{"filter":[]} -p584 -sssV_index -p585 -V.kibana -p586 -sa(dp587 -V_score -p588 -F1 -sV_type -p589 -Vsearch -p590 -sV_id -p591 -Vdns-search -p592 -sV_source -p593 -(dp594 -Vsort -p595 -(lp596 -Vtimestamp -p597 -aVdesc -p598 -asVhits -p599 -I0 -sVdescription -p600 -V -sVtitle -p601 -VDNS Requests -p602 -sVversion -p603 -I1 -sVkibanaSavedObjectMeta -p604 -(dp605 -VsearchSourceJSON -p606 -V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p607 -ssVcolumns -p608 -(lp609 -Vquery -p610 -aVqtype_name -p611 -aVanswers -p612 -aVip_src_addr -p613 -aVip_dst_addr -p614 -assV_index -p615 -V.kibana -p616 -sa(dp617 -V_score -p618 -F1 -sV_type -p619 -Vvisualization -p620 -sV_id -p621 -VFlow-Locations -p622 -sV_source -p623 -(dp624 -VvisState -p625 -V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} -p626 -sVdescription -p627 -V -sVtitle -p628 -VFlow Locations -p629 -sVuiStateJSON -p630 -V{} -p631 -sVversion -p632 -I1 -sVkibanaSavedObjectMeta -p633 -(dp634 -VsearchSourceJSON -p635 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p636 -sssV_index -p637 -V.kibana -p638 -sa(dp639 -V_score -p640 -F1 -sV_type -p641 -Vvisualization -p642 -sV_id -p643 -VUnusual-Referrers -p644 -sV_source -p645 -(dp646 -VvisState -p647 -V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} -p648 -sVdescription -p649 -V -sVtitle -p650 -VUnusual Referrers -p651 -sVuiStateJSON -p652 -V{} -p653 -sVversion -p654 -I1 -sVsavedSearchId -p655 -Vweb-search -p656 -sVkibanaSavedObjectMeta -p657 -(dp658 -VsearchSourceJSON -p659 -V{"filter":[]} -p660 -sssV_index -p661 -V.kibana -p662 -sa(dp663 -V_score -p664 -F1 -sV_type -p665 -Vvisualization -p666 -sV_id -p667 -VFrequent-DNS-Requests -p668 -sV_source -p669 -(dp670 -VvisState -p671 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} -p672 -sVdescription -p673 -V -sVtitle -p674 -VFrequent DNS Requests -p675 -sVuiStateJSON -p676 -V{} -p677 -sVversion -p678 -I1 -sVkibanaSavedObjectMeta -p679 -(dp680 -VsearchSourceJSON -p681 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p682 -sssV_index -p683 -V.kibana -p684 -sa(dp685 -V_score -p686 -F1 -sV_type -p687 -Vvisualization -p688 -sV_id -p689 -VCountry -p690 -sV_source -p691 -(dp692 -VvisState -p693 -V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p694 -sVdescription -p695 -V -sVtitle -p696 -VBy Country -p697 -sVuiStateJSON -p698 -V{} -p699 -sVversion -p700 -I1 -sVkibanaSavedObjectMeta -p701 -(dp702 -VsearchSourceJSON -p703 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p704 -sssV_index -p705 -V.kibana -p706 -sa(dp707 -V_score -p708 -F1 -sV_type -p709 -Vvisualization -p710 -sV_id -p711 -VTop-Destinations -p712 -sV_source -p713 -(dp714 -VvisState -p715 -V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} -p716 -sVdescription -p717 -V -sVtitle -p718 -VTop Destinations -p719 -sVuiStateJSON -p720 -V{} -p721 -sVversion -p722 -I1 -sVkibanaSavedObjectMeta -p723 -(dp724 -VsearchSourceJSON -p725 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p726 -sssV_index -p727 -V.kibana -p728 -sa(dp729 -V_score -p730 -F1 -sV_type -p731 -Vvisualization -p732 -sV_id -p733 -VDNS-Requests-Header -p734 -sV_source -p735 -(dp736 -VvisState -p737 -V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"} -p738 -sVdescription -p739 -V -sVtitle -p740 -VDNS Requests -p741 -sVuiStateJSON -p742 -V{} -p743 -sVversion -p744 -I1 -sVkibanaSavedObjectMeta -p745 -(dp746 -VsearchSourceJSON -p747 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p748 -sssV_index -p749 -V.kibana -p750 -sa(dp751 -V_score -p752 -F1 -sV_type -p753 -Vvisualization -p754 -sV_id -p755 -VYAF-Flows-Header -p756 -sV_source -p757 -(dp758 -VvisState -p759 -V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} -p760 -sVdescription -p761 -V -sVtitle -p762 -VYAF -p763 -sVuiStateJSON -p764 -V{} -p765 -sVversion -p766 -I1 -sVkibanaSavedObjectMeta -p767 -(dp768 -VsearchSourceJSON -p769 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p770 -sssV_index -p771 -V.kibana -p772 -sa. \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py deleted file mode 100755 index f0903acf47..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk -import cPickle as pickle -import argparse, sys, os.path -import errno -import os - - -class DashboardIndex(object): - - def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): - """ - :arg host: hostname of the node (default: localhost) - :arg port: port to use (integer, default: 9200) - :arg url_prefix: optional url prefix for elasticsearch - :arg timeout: default timeout in seconds (float, default: 10) - """ - self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) - - def get(self): - """ - Get .kibana index from Elasticsearch - """ - dotkibana = self.es.search(index='.kibana', size = 100) - return dotkibana['hits']['hits'] - - def load(self,filespec): - """ - Save Index data on local filesystem - :args filespec: path/filename for saved file - """ - data=[] - with open(filespec,'rb') as fp: - data = pickle.load(fp) - return data - - def save(self,filename,data): - """ - Save Index data on local filesystem - :args filespec: path/filename for saved file - """ - with open(filename,'wb') as fp: - pickle.dump(data,fp) - - def put(self,data): - """ - Bulk write data to Elasticsearch - :args data: data to be written (note: index name is specified in data) - """ - bulk(self.es,data) - - def main(self,args): - - if args.save: - print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) - self.save(filename=args.filespec,data=di.get()) - else: - """ - Loads Kibana Dashboard definition from disk and replaces .kibana on index - :args filespec: path/filename for saved file - """ - if not os.path.isfile(args.filespec): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) - self.es.indices.delete(index='.kibana', ignore=[400, 404]) - self.put(data=di.load(filespec=args.filespec)) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument("hostname", help="ES Hostname or IP", type=str) - parser.add_argument("port", help="ES Port", type=int) - parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) - parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") - args = parser.parse_args() - di = DashboardIndex(host=args.hostname,port=args.port) - di.main(args) diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py deleted file mode 100755 index 7913c8cbd8..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -kibana_master - -""" - - -from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script -from resource_management.libraries.functions.format import format -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate - -import errno -import os - -class Kibana(Script): - - def install(self, env): - - import params - env.set_params(params) - - Logger.info("Install Kibana Master") - - #TODO: Figure this out for all supported OSes - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Execute("echo \"[kibana-4.x]\n" - "name=Kibana repository for 4.5.x packages\n" - "baseurl=http://packages.elastic.co/kibana/4.5/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/kibana.repo") - - self.install_packages(env) - self.loadtemplate(env) - - def configure(self, env): - - import params - env.set_params(params) - - Logger.info("Configure Kibana for Metron") - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - Directory(directories, - # recursive=True, - mode=0755, - owner=params.kibana_user, - group=params.kibana_user - ) - - File("{}/kibana.yml".format(params.conf_dir), - owner=params.kibana_user, - content=InlineTemplate(params.kibana_yml_template) - ) - - def stop(self, env): - - import params - env.set_params(params) - - Logger.info("Stop Kibana Master") - - Execute("service kibana stop") - - def start(self, env): - - import params - env.set_params(params) - - self.configure(env) - - Logger.info("Start the Master") - - Execute("service kibana start") - - def restart(self,env): - - import params - env.set_params(params) - - self.configure(env) - - Logger.info("Restarting the Master") - - Execute("service kibana restart") - - def status(self, env): - - import params - env.set_params(params) - - Logger.info("Status of the Master") - - Execute("service kibana status") - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def loadtemplate(self,env): - - from dashboard.dashboardindex import DashboardIndex - - import params - env.set_params(params) - - hostname = format("{es_host}") - port = int(format("{es_port}")) - - Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname,port)) - di = DashboardIndex(host=hostname,port=port) - - #Loads Kibana Dashboard definition from disk and replaces .kibana on index - templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)),'dashboard','dashboard.p') - if not os.path.isfile(templateFile): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), templateFile) - - Logger.info("Deleting .kibana index from Elasticsearch") - - di.es.indices.delete(index='.kibana', ignore=[400, 404]) - - Logger.info("Loading .kibana index from %s" % templateFile) - - di.put(data=di.load(filespec=templateFile)) - - -if __name__ == "__main__": - Kibana().execute() diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py deleted file mode 100755 index 03fae0b759..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Kibana Params configurations - -""" - - -from resource_management.libraries.script import Script -from resource_management.libraries.functions import format -from urlparse import urlparse - -# server configurations -config = Script.get_config() - -kibana_home = '/usr/share/kibana/' -kibana_bin = '/usr/share/kibana/bin/' - -conf_dir = "/opt/kibana/config/" -kibana_user = config['configurations']['kibana-env']['kibana_user'] -user_group = config['configurations']['kibana-env']['user_group'] -log_dir = config['configurations']['kibana-env']['kibana_log_dir'] -pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] -pid_file = format("{pid_dir}/kibanasearch.pid") -es_url = config['configurations']['kibana-env']['kibana_es_url'] -parsed = urlparse(es_url) -es_host = parsed.netloc.split(':')[0] -es_port = parsed.netloc.split(':')[1] -kibana_port = config['configurations']['kibana-env']['kibana_server_port'] -hostname = config['hostname'] -java64_home = config['hostLevelParams']['java_home'] -kibana_yml_template = config['configurations']['kibana-site']['content'] diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json deleted file mode 100755 index 448e10245d..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "metron_ui", - "label": "Metron UI", - "requires_user_name": "false", - "component_name": "KIBANA_MASTER", - "url":"%@://%@:%@/", - "port":{ - "http_property": "kibana_server_port", - "http_default_port": "5601", - "https_property": "kibana_server_port", - "https_default_port": "5601", - "regex": "^(\\d+)$", - "site": "kibana-env" - } - } - ] - } -} \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml deleted file mode 100755 index 361f5f5f01..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - 2.0 - - - ELASTICSEARCH - Elasticsearch - Indexing and Search - 2.3.3 - common-services/ELASTICSEARCH/2.3.3 - - - any - - - elasticsearch-2.3.3 - - - - - - - \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml deleted file mode 100755 index 494f71b355..0000000000 --- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - 2.0 - - - KIBANA - common-services/KIBANA/4.5.1 - - - - From bad65fe5448873d9358fe86d061699b47d1f85e0 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 10:04:05 -0400 Subject: [PATCH 71/82] Updating README to include mpack information --- metron-deployment/README.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 6a44596376..7cf3765fd1 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -3,6 +3,8 @@ This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Me playbooks. These playbooks currently only target RHEL/CentOS 6.x operating systems. +In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed later. + ## Prerequisites The following tools are required to run these scripts: @@ -91,6 +93,38 @@ installed Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. This also provides a good example of how to run a full end-to-end Metron install. +## Ambari Management Pack +An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update. + +This will set up +- Metron Parsers +- Enrichment +- Indexing +- GeoIP database on MySQL +- Optional Elasticsearch +- Optional Kibana + +### Prerequisites +- A cluster managed by Ambari +- Metron RPMs available on the cluster in the /localrepo directory. See [RPM](#RPM) for further information. + +### Building Management Pack +From `metron-deployment` run +``` +mvn clean package +``` + +A tar.gz that can be used with Ambari can be found at `metron-deployment/packaging/ambari/metron-mpack/target/` + +### Installing Management Pack +Place the mpack's tar.gz onto the node running Ambari Server. From the command line on this node, run +``` +ambari-server install-mpack --mpack= --verbose +``` + +This will make the services available in Ambari in the same manner as any services in a stack, e.g. through Add Services or during cluster install. + + ## RPM RPMs can be built to install the components in metron-platform. These RPMs are built in a Docker container and placed into `target`. @@ -111,7 +145,7 @@ Components in the RPMs: ### Building RPMs From `metron-deployment` run ``` -mvn clean package +mvn clean package -Pbuild-rpms ``` The output RPM files will land in `target/RPMS/noarch`. They can be installed with the standard From 206df5f2cdf51001f4a2a9d42295268c93c22cc2 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 10:45:21 -0400 Subject: [PATCH 72/82] Cleaning up format, and making retention consistent across topics --- .../package/scripts/enrichment_commands.py | 2 +- .../0.2.0BETA/package/scripts/enrichment_master.py | 4 ++-- .../0.2.0BETA/package/scripts/indexing_commands.py | 1 + .../0.2.0BETA/package/scripts/indexing_master.py | 2 +- .../0.2.0BETA/package/scripts/metron_service.py | 11 +++++------ .../package/scripts/params/params_linux.py | 1 - .../0.2.0BETA/package/scripts/parser_commands.py | 13 ++++++++----- .../0.2.0BETA/package/scripts/parser_master.py | 4 ++-- .../0.2.0BETA/package/scripts/service_check.py | 3 ++- 9 files changed, 22 insertions(+), 19 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 8c8b0d2b3c..032add8865 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -81,7 +81,7 @@ def init_kafka_topics(self): --config retention.bytes={5}""" num_partitions = 1 replication_factor = 1 - retention_gigabytes = 10 + retention_gigabytes = int(self.__params.metron_topic_retention) retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for enrichment") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py index 8e105cb255..390576b605 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py @@ -17,9 +17,9 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import File -from resource_management.libraries.script import Script from resource_management.core.source import Template from resource_management.libraries.functions.format import format +from resource_management.libraries.script import Script from enrichment_commands import EnrichmentCommands @@ -74,7 +74,7 @@ def restart(self, env): from params import params env.set_params(params) commands = EnrichmentCommands(params) - commands.restart_enrichment_topology() + commands.restart_enrichment_topology(env) def kafkabuild(self, env, upgrade_type=None): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 885331bb99..616f2dda4b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -17,6 +17,7 @@ import os import time + from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index 8fa0afd629..b1195db4f0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -33,7 +33,7 @@ def install(self, env): Logger.info('Install RPM packages') self.install_packages(env) - def configure(self, env): + def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py index 34eb106bc8..33358f81b3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -14,14 +14,14 @@ limitations under the License. """ +import json +import subprocess + from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import format as ambari_format from resource_management.core.resources.system import Directory, File +from resource_management.core.resources.system import Execute from resource_management.core.source import InlineTemplate - -import subprocess -import json +from resource_management.libraries.functions import format as ambari_format def init_config(): @@ -59,7 +59,6 @@ def load_global_config(params): directories = [params.metron_zookeeper_config_path] Directory(directories, - # recursive=True, mode=0755, owner=params.metron_user, group=params.metron_group diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index c5e4eda365..1b75633a31 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -103,7 +103,6 @@ # for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] -hostname = config["hostname"] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py index d59fa6d685..b3fb8094fe 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py @@ -25,8 +25,10 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File + import metron_service + # Wrap major operations and functionality in this class class ParserCommands: __params = None @@ -56,7 +58,7 @@ def set_configured(self): def init_parsers(self): Logger.info( "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir, - self.__params.metron_apps_dir)) + self.__params.metron_apps_dir)) self.__params.HdfsResource(self.__params.metron_apps_dir, type="directory", action="create_on_execute", @@ -105,7 +107,7 @@ def init_kafka_topics(self): --config retention.bytes={5}""" num_partitions = 1 replication_factor = 1 - retention_gigabytes = 10 + retention_gigabytes = int(self.__params.metron_topic_retention) retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating main topics for parsers") for parser_name in self.get_parser_list(): @@ -151,7 +153,7 @@ def stop_parser_topologies(self): Execute(stop_cmd) Logger.info('Done stopping parser topologies') - def restart_parser_topologies(self,env): + def restart_parser_topologies(self, env): Logger.info('Restarting the parser topologies') self.stop_parser_topologies() attempt_count = 0 @@ -176,15 +178,16 @@ def topologies_exist(self): return True return False - def topologies_running(self,env): + def topologies_running(self, env): env.set_params(self.__params) all_running = True topologies = metron_service.get_running_topologies() for parser in self.get_parser_list(): parser_found = False + is_running = False if parser in topologies: parser_found = True - is_running = topologies[parser] in ['ACTIVE','REBALANCING'] + is_running = topologies[parser] in ['ACTIVE', 'REBALANCING'] all_running &= parser_found and is_running return all_running diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py index eb90bc6630..3758873cad 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py @@ -39,7 +39,6 @@ def install(self, env): Logger.info('Install RPM packages') self.install_packages(env) - def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) @@ -77,7 +76,7 @@ def restart(self, env): commands = ParserCommands(params) commands.restart_parser_topologies(env) - def servicechecktest(self,env): + def servicechecktest(self, env): from params import params env.set_params(params) from service_check import ServiceCheck @@ -85,5 +84,6 @@ def servicechecktest(self,env): Logger.info('Service Check Test') service_check.service_check(env) + if __name__ == "__main__": ParserMaster().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py index 1af1692f92..7dd9dfb8fb 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py @@ -20,8 +20,9 @@ from __future__ import print_function from resource_management.libraries.script import Script -from parser_commands import ParserCommands + from indexing_commands import IndexingCommands +from parser_commands import ParserCommands class ServiceCheck(Script): From 91afbe275a7550bc52a58b26794d243652b305db Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 11:48:10 -0400 Subject: [PATCH 73/82] Removing extra custom command and adding curl timeout --- .../common-services/METRON/0.2.0BETA/metainfo.xml | 12 +----------- .../0.2.0BETA/package/scripts/metron_service.py | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml index 481e468b1f..6365d5069f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -67,16 +67,6 @@ PYTHON - - - SERVICECHECKTEST - false - - - PYTHON - - - metron-parsers @@ -84,7 +74,7 @@ METRON_ENRICHMENT_MASTER - Enrichment Master + Metron Enrichment MASTER 1 true diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py index 33358f81b3..57da2c7c64 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py @@ -35,7 +35,7 @@ def init_config(): def get_running_topologies(): Logger.info('Getting Running Storm Topologies from Storm REST Server') - cmd = ambari_format('curl {storm_rest_addr}/api/v1/topology/summary') + cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary') proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() From c7773f322e5fdd80f2286a8454da66f222003fdc Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 12:40:28 -0400 Subject: [PATCH 74/82] Splitting topology and topic names, removing some hardcoding, formatting files --- .../2.3.3/package/scripts/elastic.py | 8 +-- .../2.3.3/package/scripts/elastic_master.py | 4 +- .../2.3.3/package/scripts/elastic_slave.py | 3 +- .../2.3.3/package/scripts/params.py | 2 +- .../2.3.3/package/scripts/service_check.py | 4 +- .../2.3.3/package/scripts/slave.py | 4 +- .../4.5.1/package/scripts/kibana_master.py | 54 ++++++++----------- .../KIBANA/4.5.1/package/scripts/params.py | 4 +- .../package/scripts/enrichment_commands.py | 26 ++++----- .../package/scripts/params/params_linux.py | 10 +++- .../package/scripts/params/status_params.py | 9 +++- .../templates/enrichment.properties.j2 | 12 ++--- 12 files changed, 76 insertions(+), 64 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py index bd858cd66d..a4726557bb 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py @@ -43,16 +43,16 @@ def elastic(): group=params.elastic_user ) - print "Master env: ""{}/elastic-env.sh".format(params.conf_dir) - File("{}/elastic-env.sh".format(params.conf_dir), + print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) + File("{0}/elastic-env.sh".format(params.conf_dir), owner=params.elastic_user, content=InlineTemplate(params.elastic_env_sh_template) ) configurations = params.config['configurations']['elastic-site'] - print "Master yml: ""{}/elasticsearch.yml".format(params.conf_dir) - File("{}/elasticsearch.yml".format(params.conf_dir), + print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir) + File("{0}/elasticsearch.yml".format(params.conf_dir), content=Template( "elasticsearch.master.yaml.j2", configurations=configurations), diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py index 5fc29cf32b..3ad4e1f2b0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py @@ -17,10 +17,11 @@ """ -from elastic import elastic from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script +from elastic import elastic + class Elasticsearch(Script): def install(self, env): @@ -75,5 +76,6 @@ def restart(self, env): print 'Restarting the Master' Execute(restart_cmd) + if __name__ == "__main__": Elasticsearch().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py index e65bd8f28e..4b636e886a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py @@ -17,8 +17,9 @@ """ -from resource_management.libraries.script import Script from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script + from slave import slave diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py index c3e9169416..30e88ea782 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -74,4 +74,4 @@ discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] -network_host = config['configurations']['elastic-site']['network_host'] \ No newline at end of file +network_host = config['configurations']['elastic-site']['network_host'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py index 9615d83405..e84fb010ee 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py @@ -19,11 +19,11 @@ """ from __future__ import print_function +import subprocess import sys -from resource_management.libraries.script import Script from resource_management.core.resources.system import Execute -import subprocess +from resource_management.libraries.script import Script class ServiceCheck(Script): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py index a134160e13..90d4d3d245 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py @@ -42,14 +42,14 @@ def slave(): cd_access="a" ) - File("{}/elastic-env.sh".format(params.conf_dir), + File("{0}/elastic-env.sh".format(params.conf_dir), owner=params.elastic_user, content=InlineTemplate(params.elastic_env_sh_template) ) configurations = params.config['configurations']['elastic-site'] - File("{}/elasticsearch.yml".format(params.conf_dir), + File("{0}/elasticsearch.yml".format(params.conf_dir), content=Template( "elasticsearch.slave.yaml.j2", configurations=configurations), diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py index cfa1f39453..e76ede434f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -19,29 +19,27 @@ """ +import errno +import os from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script -from resource_management.libraries.functions.format import format from resource_management.core.logger import Logger from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import Execute from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions.format import format +from resource_management.libraries.script import Script -import errno -import os class Kibana(Script): - def install(self, env): - import params env.set_params(params) Logger.info("Install Kibana Master") - #TODO: Figure this out for all supported OSes + # TODO: Figure this out for all supported OSes Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') Execute("echo \"[kibana-4.x]\n" "name=Kibana repository for 4.5.x packages\n" @@ -52,8 +50,7 @@ def install(self, env): self.install_packages(env) - def configure(self, env): - + def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) @@ -61,19 +58,18 @@ def configure(self, env): directories = [params.log_dir, params.pid_dir, params.conf_dir] Directory(directories, - # recursive=True, - mode=0755, - owner=params.kibana_user, - group=params.kibana_user - ) + # recursive=True, + mode=0755, + owner=params.kibana_user, + group=params.kibana_user + ) File("{}/kibana.yml".format(params.conf_dir), - owner=params.kibana_user, - content=InlineTemplate(params.kibana_yml_template) - ) - - def stop(self, env): + owner=params.kibana_user, + content=InlineTemplate(params.kibana_yml_template) + ) + def stop(self, env, upgrade_type=None): import params env.set_params(params) @@ -81,8 +77,7 @@ def stop(self, env): Execute("service kibana stop") - def start(self, env): - + def start(self, env, upgrade_type=None): import params env.set_params(params) @@ -92,8 +87,7 @@ def start(self, env): Execute("service kibana start") - def restart(self,env): - + def restart(self, env): import params env.set_params(params) @@ -104,7 +98,6 @@ def restart(self,env): Execute("service kibana restart") def status(self, env): - import params env.set_params(params) @@ -113,8 +106,7 @@ def status(self, env): Execute("service kibana status") @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def loadtemplate(self,env): - + def loadtemplate(self, env): from dashboard.dashboardindex import DashboardIndex import params @@ -123,11 +115,11 @@ def loadtemplate(self,env): hostname = format("{es_host}") port = int(format("{es_port}")) - Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname,port)) - di = DashboardIndex(host=hostname,port=port) + Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port)) + di = DashboardIndex(host=hostname, port=port) - #Loads Kibana Dashboard definition from disk and replaces .kibana on index - templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)),'dashboard','dashboard.p') + # Loads Kibana Dashboard definition from disk and replaces .kibana on index + templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard.p') if not os.path.isfile(templateFile): raise IOError( errno.ENOENT, os.strerror(errno.ENOENT), templateFile) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py index 03fae0b759..ef1e597ae0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py @@ -20,10 +20,10 @@ """ +from urlparse import urlparse -from resource_management.libraries.script import Script from resource_management.libraries.functions import format -from urlparse import urlparse +from resource_management.libraries.script import Script # server configurations config = Script.get_config() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 032add8865..57c601a08e 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -26,14 +26,16 @@ # Wrap major operations and functionality in this class class EnrichmentCommands: __params = None - __enrichment = None + __enrichment_topology = None + __enrichment_topic = None __configured = False def __init__(self, params): if params is None: raise ValueError("params argument is required for initialization") self.__params = params - self.__enrichment = params.metron_enrichment_topology + self.__enrichment_topology = params.metron_enrichment_topology + self.__enrichment_topology = params.metron_enrichment_topic self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) def is_configured(self): @@ -85,28 +87,28 @@ def init_kafka_topics(self): retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for enrichment") - Logger.info("Creating topic'{0}'".format(self.__enrichment)) + Logger.info("Creating topic'{0}'".format(self.__enrichment_topic)) Execute(command_template.format(self.__params.kafka_bin_dir, self.__params.zookeeper_quorum, - self.__enrichment, + self.__enrichment_topic, num_partitions, replication_factor, retention_bytes)) Logger.info("Done creating Kafka topics") def start_enrichment_topology(self): - Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment)) + Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology)) start_cmd_template = """{0}/bin/start_enrichment_topology.sh \ -s {1} \ -z {2}""" - Logger.info('Starting ' + self.__enrichment) - Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment, self.__params.zookeeper_quorum)) + Logger.info('Starting ' + self.__enrichment_topology) + Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum)) Logger.info('Finished starting enrichment topology') def stop_enrichment_topology(self): - Logger.info('Stopping ' + self.__enrichment) - stop_cmd = 'storm kill ' + self.__enrichment + Logger.info('Stopping ' + self.__enrichment_topology) + stop_cmd = 'storm kill ' + self.__enrichment_topology Execute(stop_cmd) Logger.info('Done stopping enrichment topologies') @@ -131,7 +133,7 @@ def restart_enrichment_topology(self, env): def is_topology_active(self, env): env.set_params(self.__params) - cmd_retrieve = "storm list | grep 'enrichment'" + cmd_retrieve = "storm list | grep '{0}'".format(self.__enrichment_topology) proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() Logger.info("Retrieval response is: %s" % stdout) @@ -148,7 +150,7 @@ def is_topology_active(self, env): return status in running_status_set def create_hbase_tables(self): - add_enrichment_cmd = format("echo \"create 'enrichment','t'\" | hbase shell -n") + add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf) Execute(add_enrichment_cmd, tries=3, try_sleep=5, @@ -156,7 +158,7 @@ def create_hbase_tables(self): path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) - add_threatintel_cmd = format("echo \"create 'threatintel','t'\" | hbase shell -n") + add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf) Execute(add_threatintel_cmd, tries=3, try_sleep=5, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 1b75633a31..918f8ba877 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -43,7 +43,6 @@ parsers = status_params.parsers metron_ddl_dir = metron_home + '/ddl' geo_ip_ddl = metron_ddl_dir + '/geoip_ddl.sql' -metron_enrichment_topology = status_params.metron_enrichment_topology metron_indexing_topology = status_params.metron_indexing_topology metron_user = config['configurations']['metron-env']['metron_user'] metron_group = config['configurations']['metron-env']['metron_group'] @@ -153,3 +152,12 @@ immutable_paths=get_not_managed_resources(), dfs_type=dfs_type ) + +# HBase +enrichment_table = status_params.enrichment_table +enrichment_cf = status_params.enrichment_cf +threatintel_table = status_params.threatintel_table +threatintel_cf = status_params.threatintel_cf + +metron_enrichment_topology = status_params.metron_enrichment_topology +metron_enrichment_topic = status_params.metron_enrichment_topic diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py index 498ffea7e2..7994ffe885 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py @@ -32,7 +32,14 @@ parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured' # Enrichment -metron_enrichment_topology = 'enrichments' +metron_enrichment_topology = 'enrichment' +metron_enrichment_topic = 'enrichments' + +enrichment_table = 'enrichment' +enrichment_cf = 't' +threatintel_table = 'threatintel' +threatintel_cf = 't' + mysql_process_name = 'mysqld' if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): daemon_name = 'mysql' diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 index fad8530003..bab2f52ff8 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 @@ -60,8 +60,8 @@ org.apache.metron.metrics.TelemetryIndexingBolt.fails=true ##### Host Enrichment ##### hbase.provider.impl=org.apache.metron.hbase.HTableProvider -enrichment.simple.hbase.table=enrichment -enrichment.simple.hbase.cf=t +enrichment.simple.hbase.table={{enrichment_table}} +enrichment.simple.hbase.cf={{enrichment_cf}} org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] @@ -79,10 +79,10 @@ bolt.hbase.partitioner.region.info.refresh.interval.mins=60 ##### Threat Intel ##### -threat.intel.tracker.table=threatintel -threat.intel.tracker.cf=t -threat.intel.simple.hbase.table=threatintel -threat.intel.simple.hbase.cf=t +threat.intel.tracker.table={{threatintel_table}} +threat.intel.tracker.cf={{threatintel_cf}} +threat.intel.simple.hbase.table={{threatintel_table}} +threat.intel.simple.hbase.cf={{threatintel_cf}} threat.intel.ip.table= threat.intel.ip.cf= From fd8a70746dd7f23f3e9f79e7557b735cf50c10f1 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 14:17:20 -0400 Subject: [PATCH 75/82] Fixing topic naming bug. Using Storm API for retrieving active topologies --- .../package/scripts/enrichment_commands.py | 28 ++++++++----------- .../package/scripts/indexing_commands.py | 4 +-- .../package/scripts/params/status_params.py | 2 +- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py index 57c601a08e..027ffae38d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py @@ -16,12 +16,13 @@ """ import os -import subprocess import time from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File +import metron_service + # Wrap major operations and functionality in this class class EnrichmentCommands: @@ -35,7 +36,7 @@ def __init__(self, params): raise ValueError("params argument is required for initialization") self.__params = params self.__enrichment_topology = params.metron_enrichment_topology - self.__enrichment_topology = params.metron_enrichment_topic + self.__enrichment_topic = params.metron_enrichment_topic self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) def is_configured(self): @@ -133,21 +134,14 @@ def restart_enrichment_topology(self, env): def is_topology_active(self, env): env.set_params(self.__params) - cmd_retrieve = "storm list | grep '{0}'".format(self.__enrichment_topology) - proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - Logger.info("Retrieval response is: %s" % stdout) - Logger.warning("Error response is: %s" % stderr) - - fields = stdout.split() - if len(fields) < 2: - Logger.warning("Enrichment topology is not running") - return False - - # Get the second column, which is status. We already know first column is enrichment) - status = stdout.split()[1] - running_status_set = ['ACTIVE', 'REBALANCING'] - return status in running_status_set + + active = True + topologies = metron_service.get_running_topologies() + is_running = False + if self.__enrichment_topology in topologies: + is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING'] + active &= is_running + return active def create_hbase_tables(self): add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py index 616f2dda4b..8a7e3c6cfe 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py @@ -136,7 +136,7 @@ def is_topology_active(self, env): active = True topologies = metron_service.get_running_topologies() is_running = False - if 'indexing' in topologies: - is_running = topologies['indexing'] in ['ACTIVE', 'REBALANCING'] + if self.__indexing in topologies: + is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING'] active &= is_running return active diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py index 7994ffe885..a0310d235c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py @@ -46,7 +46,7 @@ else: daemon_name = 'mysqld' -# Indexing +# ing metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' From 79a473e5bb150ca9c26cfc0585504a151390afb5 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 15 Sep 2016 14:56:09 -0400 Subject: [PATCH 76/82] Addin custom actions for loading/unloading elasticsearch templates --- .../METRON/0.2.0BETA/metainfo.xml | 19 +- .../package/files/bro_index.template | 217 ++++++++++++++++++ .../package/files/snort_index.template | 183 +++++++++++++++ .../package/files/yaf_index.template | 205 +++++++++++++++++ .../package/scripts/indexing_master.py | 44 ++++ .../package/scripts/params/params_linux.py | 5 + 6 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml index 6365d5069f..d9886fea40 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml @@ -166,11 +166,28 @@ PYTHON + + + ELASTICSEARCH_TEMPLATE_INSTALL + + + PYTHON + 600 + + + + ELASTICSEARCH_TEMPLATE_DELETE + + + PYTHON + 600 + + + metron-indexing - diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template new file mode 100644 index 0000000000..951d967349 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template @@ -0,0 +1,217 @@ +{ + "template": "bro_index*", + "mappings": { + "bro_doc": { + "_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "geo_location_point": { + "match": "enrichments:geo:*:location_point", + "match_mapping_type": "*", + "mapping": { + "type": "geo_point" + } + } + }, + { + "geo_country": { + "match": "enrichments:geo:*:country", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_city": { + "match": "enrichments:geo:*:city", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_location_id": { + "match": "enrichments:geo:*:locID", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_dma_code": { + "match": "enrichments:geo:*:dmaCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_postal_code": { + "match": "enrichments:geo:*:postalCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_latitude": { + "match": "enrichments:geo:*:latitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "geo_longitude": { + "match": "enrichments:geo:*:longitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "timestamps": { + "match": "*:ts", + "match_mapping_type": "*", + "mapping": { + "type": "date", + "format": "epoch_millis" + } + } + } + ], + "properties": { + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "source:type": { + "type": "string", + "index": "not_analyzed" + }, + "ip_dst_addr": { + "type": "ip" + }, + "ip_dst_port": { + "type": "integer" + }, + "ip_src_addr": { + "type": "ip" + }, + "ip_src_port": { + "type": "integer" + }, + "status_code": { + "type": "integer" + }, + "method": { + "type": "string", + "index": "not_analyzed" + }, + "protocol": { + "type": "string", + "index": "not_analyzed" + }, + "request_body_len": { + "type": "integer" + }, + "uri": { + "type": "string", + "index": "not_analyzed" + }, + "uid": { + "type": "string", + "index": "not_analyzed" + }, + "referrer": { + "type": "string", + "index": "not_analyzed" + }, + "trans_depth": { + "type": "integer" + }, + "host": { + "type": "string", + "index": "not_analyzed" + }, + "status_msg": { + "type": "string", + "index": "not_analyzed" + }, + "response_body_len": { + "type": "integer" + }, + "user_agent": { + "type": "string" + }, + "query": { + "type": "string", + "index": "not_analyzed" + }, + "answers": { + "type": "ip" + }, + "AA": { + "type": "boolean" + }, + "TC": { + "type": "boolean" + }, + "RA": { + "type": "boolean" + }, + "RD": { + "type": "boolean" + }, + "rejected": { + "type": "boolean" + }, + "qclass_name": { + "type": "string", + "index": "not_analyzed" + }, + "proto": { + "type": "string", + "index": "not_analyzed" + }, + "rcode": { + "type": "integer" + }, + "rcode_name": { + "type": "string", + "index": "not_analyzed" + }, + "trans_id": { + "type": "integer" + }, + "Z": { + "type": "integer" + }, + "qclass": { + "type": "integer" + }, + "qtype": { + "type": "integer" + }, + "qtype_name": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template new file mode 100644 index 0000000000..bf943dfc95 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template @@ -0,0 +1,183 @@ +{ + "template": "snort_index*", + "mappings": { + "snort_doc": { + "_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "geo_location_point": { + "match": "enrichments:geo:*:location_point", + "match_mapping_type": "*", + "mapping": { + "type": "geo_point" + } + } + }, + { + "geo_country": { + "match": "enrichments:geo:*:country", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_city": { + "match": "enrichments:geo:*:city", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_location_id": { + "match": "enrichments:geo:*:locID", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_dma_code": { + "match": "enrichments:geo:*:dmaCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_postal_code": { + "match": "enrichments:geo:*:postalCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_latitude": { + "match": "enrichments:geo:*:latitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "geo_longitude": { + "match": "enrichments:geo:*:longitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "timestamps": { + "match": "*:ts", + "match_mapping_type": "*", + "mapping": { + "type": "date", + "format": "epoch_millis" + } + } + } + ], + "properties": { + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "source:type": { + "type": "string", + "index": "not_analyzed" + }, + "ip_dst_addr": { + "type": "ip" + }, + "ip_dst_port": { + "type": "integer" + }, + "ip_src_addr": { + "type": "ip" + }, + "ip_src_port": { + "type": "integer" + }, + "dgmlen": { + "type": "integer" + }, + "ethdst": { + "type": "string", + "index": "not_analyzed" + }, + "ethlen": { + "type": "string", + "index": "not_analyzed" + }, + "ethsrc": { + "type": "string", + "index": "not_analyzed" + }, + "id": { + "type": "integer" + }, + "iplen": { + "type": "integer" + }, + "is_alert": { + "type": "boolean" + }, + "msg": { + "type": "string" + }, + "protocol": { + "type": "string", + "index": "not_analyzed" + }, + "sig_generator": { + "type": "string", + "index": "not_analyzed" + }, + "sig_id": { + "type": "integer" + }, + "sig_rev": { + "type": "string" + }, + "tcpack": { + "type": "string" + }, + "tcpflags": { + "type": "string" + }, + "tcpseq": { + "type": "string" + }, + "tcpwindow": { + "type": "string" + }, + "threat:triage:level": { + "type": "double" + }, + "tos": { + "type": "integer" + }, + "ttl": { + "type": "integer" + } + } + } + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template new file mode 100644 index 0000000000..7743afcd78 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template @@ -0,0 +1,205 @@ +{ + "template": "yaf_index*", + "mappings": { + "yaf_doc": { + "_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "geo_location_point": { + "match": "enrichments:geo:*:location_point", + "match_mapping_type": "*", + "mapping": { + "type": "geo_point" + } + } + }, + { + "geo_country": { + "match": "enrichments:geo:*:country", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_city": { + "match": "enrichments:geo:*:city", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_location_id": { + "match": "enrichments:geo:*:locID", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_dma_code": { + "match": "enrichments:geo:*:dmaCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_postal_code": { + "match": "enrichments:geo:*:postalCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_latitude": { + "match": "enrichments:geo:*:latitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "geo_longitude": { + "match": "enrichments:geo:*:longitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "timestamps": { + "match": "*:ts", + "match_mapping_type": "*", + "mapping": { + "type": "date", + "format": "epoch_millis" + } + } + } + ], + "properties": { + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "source:type": { + "type": "string", + "index": "not_analyzed" + }, + "ip_dst_addr": { + "type": "ip" + }, + "ip_dst_port": { + "type": "integer" + }, + "ip_src_addr": { + "type": "ip" + }, + "ip_src_port": { + "type": "integer" + }, + "start_time": { + "type": "date", + "format": "epoch_millis" + }, + "end_time": { + "type": "date", + "format": "epoch_millis" + }, + "duration": { + "type": "double" + }, + "rtt": { + "type": "double" + }, + "proto": { + "type": "string", + "index": "not_analyzed" + }, + "sip": { + "type": "string", + "index": "not_analyzed" + }, + "sp": { + "type": "string", + "index": "not_analyzed" + }, + "dip": { + "type": "string", + "index": "not_analyzed" + }, + "dp": { + "type": "string", + "index": "not_analyzed" + }, + "iflags": { + "type": "string", + "index": "not_analyzed" + }, + "uflags": { + "type": "string", + "index": "not_analyzed" + }, + "riflags": { + "type": "string" + }, + "ruflags": { + "type": "string", + "index": "not_analyzed" + }, + "isn": { + "type": "string", + "index": "not_analyzed" + }, + "risn": { + "type": "string", + "index": "not_analyzed" + }, + "tag": { + "type": "string", + "index": "not_analyzed" + }, + "rtag": { + "type": "string", + "index": "not_analyzed" + }, + "pkt": { + "type": "integer" + }, + "oct": { + "type": "integer" + }, + "rpkt": { + "type": "integer" + }, + "roct": { + "type": "integer" + }, + "app": { + "type": "string", + "index": "not_analyzed" + }, + "end-reason": { + "type": "string" + } + } + } + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index b1195db4f0..b4b28d9ae1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -16,6 +16,10 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.core.source import StaticFile +from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.script import Script import metron_service @@ -71,6 +75,46 @@ def restart(self, env): commands = IndexingCommands(params) commands.restart_indexing_topology(env) + def elasticsearch_template_install(self, env): + from params import params + env.set_params(params) + + File(params.bro_index_path, + mode=0755, + content=StaticFile('bro_index.template') + ) + + File(params.snort_index_path, + mode=0755, + content=StaticFile('snort_index.template') + ) + + File(params.yaf_index_path, + mode=0755, + content=StaticFile('yaf_index.template') + ) + + bro_cmd = ambari_format( + 'curl -s -XPOST http://{es_url}/_template/bro_index -d @roles/metron_elasticsearch_templates/files/es_templates/bro_index.template') + Execute(bro_cmd, logoutput=True) + snort_cmd = ambari_format( + 'curl -s -XPOST http://{es_url}/_template/snort_index -d @roles/metron_elasticsearch_templates/files/es_templates/snort_index.template') + Execute(snort_cmd, logoutput=True) + yaf_cmd = ambari_format( + 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template') + Execute(yaf_cmd, logoutput=True) + + def elasticsearch_template_delete(self, env): + from params import params + env.set_params(params) + + bro_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/bro_index*"') + Execute(bro_cmd, logoutput=True) + snort_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/snort_index*"') + Execute(snort_cmd, logoutput=True) + yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/yaf_index*"') + Execute(yaf_cmd, logoutput=True) + if __name__ == "__main__": Indexing().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 918f8ba877..253e7725d4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -161,3 +161,8 @@ metron_enrichment_topology = status_params.metron_enrichment_topology metron_enrichment_topic = status_params.metron_enrichment_topic + +# ES Templates +bro_index_path = tmp_dir + "/bro_index.template" +snort_index_path = tmp_dir + "/snort_index.template" +yaf_index_path = tmp_dir + "/yaf_index.template" From 6944a7443267cc8d23a9b06a634ba7d156b4d145 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 16 Sep 2016 13:59:01 -0400 Subject: [PATCH 77/82] Working Through Kibana. Adding folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 49e1afa317..5e570e416b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ tmp/**/* temp/** temp/**/* +repodata/ From b2bda8121005553d39013c3de879b50929c1d70f Mon Sep 17 00:00:00 2001 From: justinjleet Date: Mon, 19 Sep 2016 16:28:36 -0400 Subject: [PATCH 78/82] Allowing for configurable geoip url --- .../METRON/0.2.0BETA/configuration/metron-env.xml | 5 +++++ .../METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh | 6 ++++-- .../METRON/0.2.0BETA/package/scripts/mysql_utils.py | 2 +- .../METRON/0.2.0BETA/package/scripts/params/params_linux.py | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml index a86cf20b36..30f9f76ddf 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml @@ -92,6 +92,11 @@ metron Name of Elasticsearch Cluster + + geoip_url + http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz + Location of the GeoIP data to load. + es_url diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh index cbe14cdb24..21ced42e04 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh @@ -22,18 +22,20 @@ mysqldservice=$1 geoipscript=$2 +geoipurl=$3 # Download and extract the actual GeoIP files mkdir -p /tmp/geoip mkdir -p /var/lib/mysql-files/ +# Stage the GeoIP data pushd /tmp/geoip -curl -O http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz +curl -O ${geoipurl} tar xf GeoLiteCity-latest.tar.xz cp /tmp/geoip/*/*.csv /var/lib/mysql-files/ popd -# Load MySQL with the GeoIP data +# Load MySQL with the GeoIP data and start service service ${mysqldservice} start mysql -u root < ${geoipscript} mysql -u root -e "show databases;" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py index 4cb522194a..40b925d750 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py @@ -42,7 +42,7 @@ def mysql_configure(): content=StaticFile('createMysqlGeoIp.sh') ) - geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geo_ip_ddl}") + geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geoip_ddl} {geoip_url}") Execute(geoip_setup_cmd, tries=3, try_sleep=5, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py index 253e7725d4..5f914f3208 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py @@ -42,7 +42,8 @@ metron_home = status_params.metron_home parsers = status_params.parsers metron_ddl_dir = metron_home + '/ddl' -geo_ip_ddl = metron_ddl_dir + '/geoip_ddl.sql' +geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql' +geoip_url = config['configurations']['metron-env']['geoip_url'] metron_indexing_topology = status_params.metron_indexing_topology metron_user = config['configurations']['metron-env']['metron_user'] metron_group = config['configurations']['metron-env']['metron_group'] From 77dd4cbde0d78dbc30c8458d6715fd3f5d725b87 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 20 Sep 2016 09:43:39 -0400 Subject: [PATCH 79/82] Updating README --- metron-deployment/README.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 7cf3765fd1..078ab66846 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -3,7 +3,7 @@ This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Me playbooks. These playbooks currently only target RHEL/CentOS 6.x operating systems. -In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed later. +In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README. ## Prerequisites The following tools are required to run these scripts: @@ -29,7 +29,7 @@ structure can be found at http://docs.ansible.com/ansible/playbooks_best_practic ## Ambari The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This -section can be skipped if installing Metron on a pre-existing cluster. +section can be skipped if installing Metron on a pre-existing cluster. Currently, this playbook supports building a local development cluster running on one node but options for other types of clusters will be added in the future. @@ -105,7 +105,7 @@ This will set up - Optional Kibana ### Prerequisites -- A cluster managed by Ambari +- A cluster managed by Ambari 2.4 - Metron RPMs available on the cluster in the /localrepo directory. See [RPM](#RPM) for further information. ### Building Management Pack @@ -117,13 +117,32 @@ mvn clean package A tar.gz that can be used with Ambari can be found at `metron-deployment/packaging/ambari/metron-mpack/target/` ### Installing Management Pack +Before installing the mpack, update Storm's topology.classpath in Ambari to include '/etc/hbase/conf:/etc/hadoop/conf'. Restart Storm service. + Place the mpack's tar.gz onto the node running Ambari Server. From the command line on this node, run ``` ambari-server install-mpack --mpack= --verbose ``` This will make the services available in Ambari in the same manner as any services in a stack, e.g. through Add Services or during cluster install. +The Indexing / Parsers/ Enrichment masters should be colocated with a Kafka Broker (to create topics) and HBase client (to create the enrichment and theatintel tables). +This colocation is currently not enforced by Ambari, and should be managed by either a Service or Stack advisor as an enhancement. + +Several configuration parameters will need to be filled in, and should be pretty self explanatory (primarily a couple of Elasticsearch configs, and the Storm REST URL). Examples are provided in the descriptions on Ambari. +Notably, the URL for the GeoIP database that is preloaded (and is prefilled by default) can be set to use a `file://` location + +#### Offline installation +Currently there is only one point that would reach out to the internet during an install. This is the URL for the GeoIP database information that is preloaded into MySQL. +The RPMs DO NOT reach out to the internet (because there is currently no hosting for them). They look on the local filesystem in `/localrepo`. + +### Current Limitations +- There is currently no hosting for RPMs remotely. They will have to be built locally. +- Colocation of appropriate services should be enforced by Ambari. See [#Installing Management Pack] for more details. +- Storm's topology.classpath is not updated with the Metron service install and needs to be updated separately. +- Several configuration parameters used when installing the Metron service could (and should) be grabbed from Ambari. Install will require them to be manually entered. +- MySQL install should be optional (and allow for using an existing instance). +- Need to handle upgrading Metron ## RPM RPMs can be built to install the components in metron-platform. These RPMs are built in a Docker container and placed into `target`. @@ -156,3 +175,4 @@ rpm -i ## TODO - migrate existing MySQL/GeoLite playbook - Support Ubuntu deployments + From aa4991d386f5f4988f501f7e3a1faae669b14d4b Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 21 Sep 2016 08:48:19 -0400 Subject: [PATCH 80/82] Updating README --- metron-deployment/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 078ab66846..f5deeab657 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -137,11 +137,14 @@ Currently there is only one point that would reach out to the internet during an The RPMs DO NOT reach out to the internet (because there is currently no hosting for them). They look on the local filesystem in `/localrepo`. ### Current Limitations +There are a set of limitations that should be addressed based to improve the current state of the mpacks. + +- MySQL install should be optional (and allow for using an existing instance). +- MySQL should not be installed on a node already running a MySQL instance (e.g. an Ambari Server using MySQL as its database). - There is currently no hosting for RPMs remotely. They will have to be built locally. - Colocation of appropriate services should be enforced by Ambari. See [#Installing Management Pack] for more details. - Storm's topology.classpath is not updated with the Metron service install and needs to be updated separately. - Several configuration parameters used when installing the Metron service could (and should) be grabbed from Ambari. Install will require them to be manually entered. -- MySQL install should be optional (and allow for using an existing instance). - Need to handle upgrading Metron ## RPM From a1d5e55366802eca8f3a612e25948f69482aa019 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 21 Sep 2016 08:55:56 -0400 Subject: [PATCH 81/82] Updating README --- metron-deployment/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index f5deeab657..ea7167ed57 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -131,6 +131,8 @@ This colocation is currently not enforced by Ambari, and should be managed by ei Several configuration parameters will need to be filled in, and should be pretty self explanatory (primarily a couple of Elasticsearch configs, and the Storm REST URL). Examples are provided in the descriptions on Ambari. Notably, the URL for the GeoIP database that is preloaded (and is prefilled by default) can be set to use a `file://` location +After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates. + #### Offline installation Currently there is only one point that would reach out to the internet during an install. This is the URL for the GeoIP database information that is preloaded into MySQL. From d44cc4efc6bddfddb114e6994c4794d21cde3f08 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 23 Sep 2016 13:00:27 -0400 Subject: [PATCH 82/82] Fixing template loading for ES --- metron-deployment/README.md | 2 +- .../resources/common-services/KIBANA/4.5.1/metainfo.xml | 2 +- .../KIBANA/4.5.1/package/scripts/kibana_master.py | 2 +- .../METRON/0.2.0BETA/package/scripts/indexing_master.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index ea7167ed57..eceeb80fef 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -131,7 +131,7 @@ This colocation is currently not enforced by Ambari, and should be managed by ei Several configuration parameters will need to be filled in, and should be pretty self explanatory (primarily a couple of Elasticsearch configs, and the Storm REST URL). Examples are provided in the descriptions on Ambari. Notably, the URL for the GeoIP database that is preloaded (and is prefilled by default) can be set to use a `file://` location -After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates. +After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates. Similar to this, a custom Kibana action to Load Template is available. #### Offline installation Currently there is only one point that would reach out to the internet during an install. This is the URL for the GeoIP database information that is preloaded into MySQL. diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml index d14afbf5e8..f59109c8c7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml @@ -36,7 +36,7 @@ - LOADTEMPLATE + LOAD_TEMPLATE false diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py index e76ede434f..10e3a95969 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -106,7 +106,7 @@ def status(self, env): Execute("service kibana status") @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def loadtemplate(self, env): + def load_template(self, env): from dashboard.dashboardindex import DashboardIndex import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py index b4b28d9ae1..eabf5a1a19 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py @@ -95,13 +95,13 @@ def elasticsearch_template_install(self, env): ) bro_cmd = ambari_format( - 'curl -s -XPOST http://{es_url}/_template/bro_index -d @roles/metron_elasticsearch_templates/files/es_templates/bro_index.template') + 'curl -s -XPOST http://{es_url}/_template/bro_index -d @{bro_index_path}') Execute(bro_cmd, logoutput=True) snort_cmd = ambari_format( - 'curl -s -XPOST http://{es_url}/_template/snort_index -d @roles/metron_elasticsearch_templates/files/es_templates/snort_index.template') + 'curl -s -XPOST http://{es_url}/_template/snort_index -d @{snort_index_path}') Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format( - 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template') + 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @{yaf_index_path}') Execute(yaf_cmd, logoutput=True) def elasticsearch_template_delete(self, env):