From 5e7afa0d61766c5bcd45e19143a4568a9da3779e Mon Sep 17 00:00:00 2001 From: nickallen Date: Wed, 19 Sep 2018 14:44:29 -0400 Subject: [PATCH 1/5] Add ability to easily deploy components for testing pcap capture --- .../ansible/playbooks/sensor_install.yml | 15 ++++++++++++++- .../ansible/roles/librdkafka/defaults/main.yml | 4 ++-- .../roles/librdkafka/tasks/librdkafka.yml | 2 +- .../files/example.pcap | Bin .../ansible/roles/pcap_replay/tasks/main.yml | 1 + .../tasks/pcap.yml | 0 .../ansible/roles/pycapa/templates/pycapa | 8 +++++--- .../roles/sensor-test-mode/tasks/main.yml | 5 +---- 8 files changed, 24 insertions(+), 11 deletions(-) rename metron-deployment/ansible/roles/{sensor-test-mode => pcap_replay}/files/example.pcap (100%) rename metron-deployment/ansible/roles/{sensor-test-mode => pcap_replay}/tasks/pcap.yml (100%) diff --git a/metron-deployment/ansible/playbooks/sensor_install.yml b/metron-deployment/ansible/playbooks/sensor_install.yml index a683ccb714..c2a1979d40 100644 --- a/metron-deployment/ansible/playbooks/sensor_install.yml +++ b/metron-deployment/ansible/playbooks/sensor_install.yml @@ -37,7 +37,7 @@ roles: - { role: ambari_gather_facts, tags: ['always'] } - { role: tap_interface, tags: ['tap'] } - - { role: pycapa, tags: ['pycapa'] } + - { role: pcap_replay, tags: ['pcap-replay'] } - { role: bro, tags: ['bro'] } - { role: snort, tags: ['snort'] } - { role: yaf, tags: ['yaf'] } @@ -46,6 +46,19 @@ tags: - sensors +# +# deploys all components required to simulate and then capture packets +# +- hosts: sensors + become: true + roles: + - { role: ambari_gather_facts } + - { role: tap_interface } + - { role: pcap_replay } + - { role: pycapa } + tags: + - pcap + # # sensor stubs - rather than deploying the sensors, deploy ligher weight # stubs that mimic the behavior of the real sensors diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml index 063c22fde2..6e25cb36d8 100644 --- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml +++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml @@ -15,6 +15,6 @@ # limitations under the License. # --- -librdkafka_version: 0.9.4 -librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz +librdkafka_version: 0.11.5 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz librdkafka_home: /usr/local diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml index a7971d639b..65132e7def 100644 --- a/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml +++ b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml @@ -15,7 +15,7 @@ # limitations under the License. # --- -- name: Download librdkafka +- name: "Download librdkafka {{ librdkafka_version }}" get_url: url: "{{ librdkafka_url }}" dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" diff --git a/metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap b/metron-deployment/ansible/roles/pcap_replay/files/example.pcap similarity index 100% rename from metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap rename to metron-deployment/ansible/roles/pcap_replay/files/example.pcap diff --git a/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml index bdc2a0bf10..a1bbfd0c67 100644 --- a/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml +++ b/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml @@ -17,4 +17,5 @@ --- - include: dependencies.yml - include: tcpreplay.yml +- include: pcap.yml - include: service.yml diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/pcap.yml similarity index 100% rename from metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml rename to metron-deployment/ansible/roles/pcap_replay/tasks/pcap.yml diff --git a/metron-deployment/ansible/roles/pycapa/templates/pycapa b/metron-deployment/ansible/roles/pycapa/templates/pycapa index ce87477434..bf43af02d4 100644 --- a/metron-deployment/ansible/roles/pycapa/templates/pycapa +++ b/metron-deployment/ansible/roles/pycapa/templates/pycapa @@ -29,7 +29,9 @@ LOGFILE="{{ pycapa_log }}" EXTRA_ARGS="${@:2}" DAEMON_PATH="{{ pycapa_home }}" -export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64 +LIBRDKAKFA_LIB={{ librdkafka_home }}/lib +PYTHON27_LIB={{ python27_home }}/usr/lib64 +export LD_LIBRARY_PATH=$PYTHON27_LIB:$LIBRDKAKFA_LIB case "$1" in @@ -47,8 +49,8 @@ case "$1" in DAEMON_PATH="{{ pycapa_home }}" DAEMON="{{ pycapa_bin }}/pycapa" DAEMONOPTS+=" --producer " - DAEMONOPTS+=" --kafka {{ kafka_broker_url }}" - DAEMONOPTS+=" --topic {{ pycapa_topic }}" + DAEMONOPTS+=" --kafka-broker {{ kafka_broker_url }}" + DAEMONOPTS+=" --kafka-topic {{ pycapa_topic }}" DAEMONOPTS+=" --interface {{ pycapa_sniff_interface }}" DAEMONOPTS+=" $EXTRA_ARGS" diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml b/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml index 24ca87e41b..0b07a3ab53 100644 --- a/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml +++ b/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml @@ -15,11 +15,8 @@ # limitations under the License. # --- -- include: pcap.yml - when: install_pcap_replay - - include: yaf.yml when: install_yaf - + - include: snort.yml when: install_snort From f6b82d0199927df0c81d84390f3987171a0321c4 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Sep 2018 15:58:49 -0400 Subject: [PATCH 2/5] Added instructions in README --- .../development/centos6/README.md | 89 ++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index 0e38d5abf3..0022ed9180 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -90,16 +90,85 @@ Connecting to the host through SSH is as simple as running the following command vagrant ssh ``` -### Working with Metron +### Advanced Deployments -In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components. +In addition to running the entire provisioning play book, you can also use Ansible tags to limit the scope of what is deployed or deploy specific components. -``` -vagrant --ansible-tags="sensor-stubs" provision -``` +If you have already built Metron, you can skip the build step when deploying the development environment. + + 1. Deploy the development environment without re-building Metron. + ``` + vagrant --ansible-skip-tags="build,sensors,pcap" up + ``` + +If you want to deploy the components required to generate and capture network packets. + + 1. Deploy the development environment. + ``` + vagrant up + ``` + + 1. Deploy the additional components to replay and capture packets. + ``` + vagrant --ansible-tags="pcap" provision + ``` + + 1. Stop the Parser, Enrichment, Indexing, and Profiler topologies to free-up resources. + + 1. Connect to the development VM. + ``` + vagrant ssh + sudo su - + source /etc/default/metron + ``` + + 1. Install Wireshark. + ``` + yum -y install wireshark + ``` + + 1. Start the Packet Replay service. + ``` + service pcap-replay start + ``` + + 1. Start Pycapa which captures those packets and pushes them to Kafka. + ``` + service pycapa start + ``` + + 1. Start the Pcap topology. + ``` + $METRON_HOME/bin/start_pcap_topology.sh + ``` + +If you want to deploy Bro, Snort, and YAF in your development environment, run the following commands. + + 1. Deploy the development environment. + ``` + vagrant up + ``` + + 1. Deploy the additional sensors. + ``` + vagrant --ansible-tags="sensors" provision + ``` + + 1. Connect to the development VM. + ``` + vagrant ssh + sudo su - + ``` + + 1. Stop the Sensor Stubs. + ``` + service sensor-stubs stop + ``` + + 1. Start the Packet Replay service. + ``` + service pcap-replay start + ``` + + 1. Ensure that each of the sensors are running. -Tags are listed in the playbooks, some frequently used tags: -+ `hdp-install` - Install HDP -+ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. -+ `sensor-stubs` - Deploy and start the sensor stubs. From c15c2d902f028e84f1959c06377cb68a3c7b8111 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Sep 2018 16:12:13 -0400 Subject: [PATCH 3/5] Fixing README formatting --- .../development/centos6/README.md | 106 ++++++++++-------- 1 file changed, 57 insertions(+), 49 deletions(-) diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index 0022ed9180..b8e4b50716 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -20,13 +20,15 @@ Metron on CentOS 6 This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running CentOS 6. -Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. - -Getting Started ---------------- +* [Prerequisites](#prerequisites) +* [Deploy Metron](#deploy-metron) +* [Explore Metron](#explore-metron) +* [Advanced Deployments](#advanced-deployments) ### Prerequisites +Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. + The computer used to deploy Apache Metron will need to have the following components installed. - [Ansible](https://github.com/ansible/ansible) (2.0.0.2, 2.2.2.0, or 2.5.0) @@ -86,89 +88,95 @@ Navigate to the following resources to explore your newly minted Apache Metron e * [Ambari](http://node1:8080) credentials: admin/admin Connecting to the host through SSH is as simple as running the following command. -``` -vagrant ssh -``` + ``` + vagrant ssh + ``` ### Advanced Deployments In addition to running the entire provisioning play book, you can also use Ansible tags to limit the scope of what is deployed or deploy specific components. +#### Avoid an Extra Build + If you have already built Metron, you can skip the build step when deploying the development environment. 1. Deploy the development environment without re-building Metron. - ``` - vagrant --ansible-skip-tags="build,sensors,pcap" up - ``` + ``` + vagrant --ansible-skip-tags="build,sensors,pcap" up + ``` + +#### Deploy Packet Capture If you want to deploy the components required to generate and capture network packets. 1. Deploy the development environment. - ``` - vagrant up - ``` + ``` + vagrant up + ``` 1. Deploy the additional components to replay and capture packets. - ``` - vagrant --ansible-tags="pcap" provision - ``` + ``` + vagrant --ansible-tags="pcap" provision + ``` 1. Stop the Parser, Enrichment, Indexing, and Profiler topologies to free-up resources. 1. Connect to the development VM. - ``` - vagrant ssh - sudo su - - source /etc/default/metron - ``` + ``` + vagrant ssh + sudo su - + source /etc/default/metron + ``` 1. Install Wireshark. - ``` - yum -y install wireshark - ``` + ``` + yum -y install wireshark + ``` 1. Start the Packet Replay service. - ``` - service pcap-replay start - ``` + ``` + service pcap-replay start + ``` 1. Start Pycapa which captures those packets and pushes them to Kafka. - ``` - service pycapa start - ``` + ``` + service pycapa start + ``` 1. Start the Pcap topology. - ``` - $METRON_HOME/bin/start_pcap_topology.sh - ``` + ``` + $METRON_HOME/bin/start_pcap_topology.sh + ``` + +#### Deploy Sensors If you want to deploy Bro, Snort, and YAF in your development environment, run the following commands. 1. Deploy the development environment. - ``` - vagrant up - ``` + ``` + vagrant up + ``` 1. Deploy the additional sensors. - ``` - vagrant --ansible-tags="sensors" provision - ``` + ``` + vagrant --ansible-tags="sensors" provision + ``` 1. Connect to the development VM. - ``` - vagrant ssh - sudo su - - ``` + ``` + vagrant ssh + sudo su - + ``` 1. Stop the Sensor Stubs. - ``` - service sensor-stubs stop - ``` + ``` + service sensor-stubs stop + ``` 1. Start the Packet Replay service. - ``` - service pcap-replay start - ``` + ``` + service pcap-replay start + ``` 1. Ensure that each of the sensors are running. From 33de7740235a17f9cf35b060c6d75ee4395c6786 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Sep 2018 16:13:53 -0400 Subject: [PATCH 4/5] Fixing README formatting --- metron-deployment/development/centos6/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index b8e4b50716..a224427818 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -25,7 +25,7 @@ This project fully automates the provisioning and deployment of Apache Metron an * [Explore Metron](#explore-metron) * [Advanced Deployments](#advanced-deployments) -### Prerequisites +## Prerequisites Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. @@ -46,7 +46,7 @@ Running the following script can help validate whether you have all the prerequi metron-deployment/scripts/platform-info.sh ``` -#### How do I install these on MacOS? +### How do I install these on MacOS? Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project. @@ -63,7 +63,7 @@ Any platform that supports these tools is suitable, but the following instructio open /Applications/Docker.app ``` -### Deploy Metron +## Deploy Metron 1. Ensure that the Docker service is running. @@ -80,7 +80,7 @@ Any platform that supports these tools is suitable, but the following instructio vagrant provision ``` -### Explore Metron +## Explore Metron Navigate to the following resources to explore your newly minted Apache Metron environment. @@ -92,11 +92,11 @@ Connecting to the host through SSH is as simple as running the following command vagrant ssh ``` -### Advanced Deployments +## Advanced Deployments In addition to running the entire provisioning play book, you can also use Ansible tags to limit the scope of what is deployed or deploy specific components. -#### Avoid an Extra Build +### Avoid an Extra Build If you have already built Metron, you can skip the build step when deploying the development environment. @@ -105,7 +105,7 @@ If you have already built Metron, you can skip the build step when deploying the vagrant --ansible-skip-tags="build,sensors,pcap" up ``` -#### Deploy Packet Capture +### Deploy Packet Capture If you want to deploy the components required to generate and capture network packets. @@ -148,7 +148,7 @@ If you want to deploy the components required to generate and capture network pa $METRON_HOME/bin/start_pcap_topology.sh ``` -#### Deploy Sensors +### Deploy Sensors If you want to deploy Bro, Snort, and YAF in your development environment, run the following commands. From 1a561f013ce0cdb0b48a73a6e374cb7dc6acf6ba Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Sep 2018 16:19:34 -0400 Subject: [PATCH 5/5] By default, we do not want the pcap stuff deployed --- metron-deployment/development/centos6/Vagrantfile | 2 +- metron-deployment/development/ubuntu14/Vagrantfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/development/centos6/Vagrantfile b/metron-deployment/development/centos6/Vagrantfile index d0b70514bf..b8eab5ae70 100644 --- a/metron-deployment/development/centos6/Vagrantfile +++ b/metron-deployment/development/centos6/Vagrantfile @@ -17,7 +17,7 @@ require 'getoptlong' ansibleTags='' -ansibleSkipTags='sensors' +ansibleSkipTags='sensors,pcap' begin opts = GetoptLong.new( diff --git a/metron-deployment/development/ubuntu14/Vagrantfile b/metron-deployment/development/ubuntu14/Vagrantfile index cfa3cdfca1..18e7391bee 100644 --- a/metron-deployment/development/ubuntu14/Vagrantfile +++ b/metron-deployment/development/ubuntu14/Vagrantfile @@ -17,7 +17,7 @@ require 'getoptlong' ansibleTags='' -ansibleSkipTags='sensors' +ansibleSkipTags='sensors,pcap' begin opts = GetoptLong.new(