From 0bf19975aae08bfc0686fc4537c2f94d90acd0f4 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 13 Oct 2018 13:21:39 -0400 Subject: [PATCH 1/4] METRON-1826: Update librdkafka for metron-bro-plugin-kafka and pycapa --- metron-deployment/ansible/roles/librdkafka/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml index 063c22fde2..6e25cb36d8 100644 --- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml +++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml @@ -15,6 +15,6 @@ # limitations under the License. # --- -librdkafka_version: 0.9.4 -librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz +librdkafka_version: 0.11.5 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz librdkafka_home: /usr/local From 10a6c2069333434fef571adf35073599c467798c Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Mon, 15 Oct 2018 10:42:41 -0400 Subject: [PATCH 2/4] Update devtoolset to 6 --- metron-deployment/ansible/roles/bro/tasks/bro.yml | 4 ++-- metron-deployment/ansible/roles/bro/tasks/dependencies.yml | 4 ++-- .../ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml index 222ef0ec57..9ba3ffa6b2 100644 --- a/metron-deployment/ansible/roles/bro/tasks/bro.yml +++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml @@ -30,8 +30,8 @@ - name: Compile and Install bro shell: "{{ item }}" environment: - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + CXX: /opt/rh/devtoolset-6/root/usr/bin/g++ + CC: /opt/rh/devtoolset-6/root/usr/bin/gcc args: chdir: "/tmp/bro-{{ bro_version }}" creates: "{{ bro_home }}/bin/bro" diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml index 3cd3baef03..f8b1e28e78 100644 --- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml +++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml @@ -38,8 +38,8 @@ - perl - crontabs - net-tools - - devtoolset-4-gcc - - devtoolset-4-gcc-c++ + - devtoolset-6-gcc + - devtoolset-6-gcc-c++ - python27 - rh-git29 register: result diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml index 7043387280..ec7af972bc 100644 --- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml +++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml @@ -30,8 +30,8 @@ environment: PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}" - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + CXX: /opt/rh/devtoolset-6/root/usr/bin/g++ + CC: /opt/rh/devtoolset-6/root/usr/bin/gcc command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}" - name: Configure bro-kafka plugin From 9b5fc3c11da46b4dd977a42895635163f1adc4df Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Mon, 22 Oct 2018 12:07:19 -0400 Subject: [PATCH 3/4] Improve documentation --- metron-deployment/development/centos6/README.md | 14 ++++++++++++-- metron-deployment/development/ubuntu14/README.md | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index e873452554..1b4752667c 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -97,8 +97,18 @@ In addition to re-running the entire provisioning play book, you may now re-run vagrant --ansible-tags="sensor-stubs" provision ``` -Tags are listed in the playbooks, some frequently used tags: +Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. ++ `sensors` - Deploy the sensors.\* + `sensor-stubs` - Deploy and start the sensor stubs. + +\* In order to start the bro, yaf, and snort sensors, and send traffic through them (via the `pcap-replay` service) you will need to do the following after your `provision` command is complete: +``` +vagrant ssh +sudo su - +service pcap-replay restart +service yaf restart +service snortd restart +service snort-producer restart +``` diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md index 494937e6de..5268b137f3 100644 --- a/metron-deployment/development/ubuntu14/README.md +++ b/metron-deployment/development/ubuntu14/README.md @@ -98,8 +98,18 @@ In addition to re-running the entire provisioning play book, you may now re-run vagrant --ansible-tags="sensor-stubs" provision ``` -Tags are listed in the playbooks, some frequently used tags: +Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. ++ `sensors` - Deploy the sensors.\* + `sensor-stubs` - Deploy and start the sensor stubs. + +\* In order to start the bro, yaf, and snort sensors, and send traffic through them (via the `pcap-replay` service) you will need to do the following after your `provision` command is complete: +``` +vagrant ssh +sudo su - +service pcap-replay restart +service yaf restart +service snortd restart +service snort-producer restart +``` From 4c8506cd6aaa5a06ebef31545a1b61c9ac8d87ee Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Mon, 22 Oct 2018 15:37:24 -0400 Subject: [PATCH 4/4] Add more detail about how the sensors tag works --- metron-deployment/development/centos6/README.md | 16 ++++++++++++++-- metron-deployment/development/ubuntu14/README.md | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index 1b4752667c..99ec967e84 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -100,10 +100,21 @@ vagrant --ansible-tags="sensor-stubs" provision Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy the sensors.\* ++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag) + `sensor-stubs` - Deploy and start the sensor stubs. -\* In order to start the bro, yaf, and snort sensors, and send traffic through them (via the `pcap-replay` service) you will need to do the following after your `provision` command is complete: +#### Sensors + +By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run. + +From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command: + +``` +vagrant --ansible-skip-tags="sensor-stubs" up +``` + +This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps: + ``` vagrant ssh sudo su - @@ -112,3 +123,4 @@ service yaf restart service snortd restart service snort-producer restart ``` + diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md index 5268b137f3..53bea677ab 100644 --- a/metron-deployment/development/ubuntu14/README.md +++ b/metron-deployment/development/ubuntu14/README.md @@ -101,10 +101,21 @@ vagrant --ansible-tags="sensor-stubs" provision Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy the sensors.\* ++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag) + `sensor-stubs` - Deploy and start the sensor stubs. -\* In order to start the bro, yaf, and snort sensors, and send traffic through them (via the `pcap-replay` service) you will need to do the following after your `provision` command is complete: +#### Sensors + +By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run. + +From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command: + +``` +vagrant --ansible-skip-tags="sensor-stubs" up +``` + +This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps: + ``` vagrant ssh sudo su - @@ -113,3 +124,4 @@ service yaf restart service snortd restart service snort-producer restart ``` +