Permalink
Switch branches/tags
Nothing to show
Find file Copy path
9713b28 Aug 13, 2018
1 contributor

Users who have contributed to this file

executable file 326 lines (274 sloc) 11.3 KB
#!/bin/bash
#
# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
COMMON="/usr/sbin/so-common"
if ! [ -f $COMMON ]; then
echo "$COMMON not found."
echo "This script requires Security Onion Elastic Stack Release Candidate 2 (14.04.5.8 ISO) or later."
exit
fi
source $COMMON
CONF="/etc/nsm/securityonion.conf"
if ! [ -f $CONF ]; then
echo "$CONF not found."
echo "Please run Setup."
exit
fi
source $CONF
SENSORTAB="/etc/nsm/sensortab"
if ! [ -f $SENSORTAB ]; then
echo "$SENSORTAB not found."
echo "This machine must be configured for network sensor services."
exit
fi
SENSOR=`grep -v "^#" $SENSORTAB |head -1 |awk '{print $1}'`
SENSORCONF="/etc/nsm/$SENSOR/sensor.conf"
if ! [ -f $SENSORCONF ]; then
echo "$SENSORCONF not found."
echo "This machine must be configured for network sensor services to continue."
exit
fi
source $SENSORCONF
SGUILDCONF="/etc/sguild/sguild.conf"
if ! [ -f $SGUILDCONF ]; then
echo "$SGUILDCONF not found."
echo "This machine must be running sguild to continue."
exit
fi
SYSLOGCONF="/etc/syslog-ng/syslog-ng.conf"
if ! [ -f $SYSLOGCONF ]; then
echo "$SYSLOGCONF not found."
echo "This machine must be running syslog-ng to continue."
exit
fi
LOGSTASH_SNORT=/etc/logstash/conf.d/1033_preprocess_snort.conf
if ! [ -f $LOGSTASH_SNORT ]; then
echo "$LOGSTASH_SNORT not found."
echo "Exiting."
exit
fi
# check to see if this is a sensor before continuing
if [ -f /root/.ssh/securityonion_ssh.conf ]; then
echo "This machine appears to be a sensor connected to a distributed deployment."
echo "This script was designed for standalone systems designated for so-import-pcap."
exit
fi
# display warnings every time
cat << EOF
so-import-pcap
This is a quick and dirty EXPERIMENTAL script that will import one or more pcaps into Security Onion and preserve original timestamps.
It will do the following:
- stop and disable Curator to avoid closing old indices
- stop and disable all active sniffing processes (Bro, Snort, Suricata, and netsniff-ng)
- stop and disable ossec_agent
- reconfigure and restart sguild, syslog-ng, and Logstash where necessary
- generate IDS alerts using Snort or Suricata
- generate Bro logs
- store IDS alerts and Bro logs with original timestamps
- split traffic into separate daily pcaps and store them where sguil's pcap_agent can find them
Requirements:
- You must be running at least Security Onion Elastic Stack Release Candidate 2 (14.04.5.8 ISO).
- You must have a sniffing interface defined (you can choose Evaluation Mode in the Setup wizard).
Warnings:
- Do NOT run this on a production deployment. It is designed for standalone systems designated for so-import-pcap.
- If you're running in a VM with snapshot capability, you might want to take a snapshot before this program makes changes.
Reverting System Changes:
- If you take a VM snapshot before this program makes changes, then just revert to snapshot.
- Otherwise, you can re-run Setup and it should overwrite all modified files to revert the system to normal operation.
EOF
# display usage if no valid pcap files are passed
function usage {
cat << EOF
Usage:
Please supply at least one pcap file.
For example, to import a single pcap named import.pcap:
so-import-pcap import.pcap
To import multiple pcaps:
so-import-pcap import1.pcap import2.pcap
EOF
}
# if no parameters supplied, display usage
if [ $# -eq 0 ]; then
usage
exit
fi
# verify that all parameters are files
for i in $@; do
if ! [ -f $i ]; then
usage
echo "$i is not a valid file!"
exit
fi
done
# prompt user before making changes
cat << EOF
If this is the first time so-import-pcap has been run on this system, then it will make changes to the system.
Press Enter to continue or Ctrl-c to cancel.
EOF
read input
# create temp pcap and set permissions
echo "Please wait while..."
echo "...creating temp pcap for processing."
PCAP=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
if ! mergecap -w $PCAP $@; then
echo "Error while merging!"
exit
fi
chmod 444 $PCAP
# check to see if file is a valid PCAP
capinfos $PCAP 2>&1 | grep "The file appears to be damaged or corrupt." && exit
# make sure sguild is running with DEBUG 2
if ! grep "set DEBUG 2" $SGUILDCONF >/dev/null 2>&1; then
echo "...setting sguild debug to 2 and restarting sguild."
sed -i 's|set DEBUG.*$|set DEBUG 2|g' $SGUILDCONF
/usr/sbin/nsm_server_ps-restart >/dev/null 2>&1
fi
# make sure syslog-ng is configured to pick up sguild logs
if ! grep "/var/log/nsm/securityonion/sguild.log" $SYSLOGCONF >/dev/null 2>&1; then
echo "...configuring syslog-ng to pick up sguild logs."
cat << EOF >> $SYSLOGCONF
# added by so-import-pcap
source s_sguil { file("/var/log/nsm/securityonion/sguild.log" program_override("snort")); };
filter f_sguil { message("Alert Received"); };
log {
source(s_sguil);
filter(f_sguil);
destination(d_logstash);
};
EOF
service syslog-ng restart >/dev/null 2>&1
fi
# disable barnyard syslog output
if grep "^output alert_syslog: LOG_LOCAL6 LOG_ALERT$" /etc/nsm/$SENSOR/barnyard*.conf >/dev/null 2>&1; then
echo "...disabling syslog output in barnyard."
sed -i 's|^output alert_syslog: LOG_LOCAL6 LOG_ALERT$|#output alert_syslog: LOG_LOCAL6 LOG_ALERT|g' /etc/nsm/$SENSOR/barnyard*.conf
nsm_sensor_ps-restart --only-barnyard2 >/dev/null 2>&1
fi
# make sure Logstash is configured to parse sguild logs
if ! grep "DATA:timestamp" $LOGSTASH_SNORT > /dev/null 2>&1; then
echo -n "...configuring logstash to parse sguild logs (this may take a few minutes, but should only need to be done once)..."
sed -i 's|"message", "%{GREEDYDATA:alert}"]|"message", "\\A%{TIME} pid\\(%{INT}\\) Alert Received: %{INT} %{INT:priority} %{DATA:classification} %{DATA:interface} \\{%{DATA:timestamp}} %{INT} %{INT} \\{%{DATA:alert}\} %{IP:source_ip} %{IP:destination_ip} %{INT:protocol} %{INT:source_port} %{INT:destination_port} %{INT:gid} %{INT:sid} %{INT:rev} %{INT} %{INT}\\Z",\n "message", "%{GREEDYDATA:alert}"]\n }\n\n if [timestamp] {\n mutate {\n add_field => { "logstash_timestamp" => "%{@timestamp}" }\n }\n mutate {\n convert => { "logstash_timestamp" => "string" }\n }\n date {\n match => [ "timestamp", "yyyy-MM-dd HH:mm:ss" ]\n }\n mutate {\n rename => { "logstash_timestamp" => "timestamp" }\n }|g' $LOGSTASH_SNORT
docker restart so-logstash >/dev/null 2>&1
if (timeout 10m tail -F -n0 "/var/log/logstash/logstash.log" &) | grep -q "Pipelines running" ; then
echo "done."
else
echo "failed."
exit
fi
fi
# stop curator if running
if docker ps |grep curator >/dev/null 2>&1; then
echo "...stopping curator."
docker stop so-curator >/dev/null 2>&1
fi
# disable curator
if [ "$CURATOR_ENABLED" = "yes" ]; then
echo "...disabling curator."
sed -i 's|CURATOR_ENABLED="yes"|CURATOR_ENABLED="no"|g' $CONF
fi
# stop ossec_agent if running
if pgrep -f /usr/bin/ossec_agent >/dev/null 2>&1; then
echo "...stopping ossec_agent."
/usr/sbin/nsm_sensor_ps-stop --only-ossec-agent >/dev/null 2>&1
fi
# disable ossec_agent
if [ "$OSSEC_AGENT_ENABLED" = "yes" ]; then
echo "...disabling ossec_agent."
sed -i 's|OSSEC_AGENT_ENABLED=yes|OSSEC_AGENT_ENABLED=no|g' $CONF
fi
# stop bro if running
if pgrep -f /opt/bro/share/broctl/scripts/run-bro >/dev/null 2>&1; then
echo "...stopping Bro sniffing process."
/usr/sbin/nsm_sensor_ps-stop --only-bro >/dev/null 2>&1
fi
# disable bro
if [ "$BRO_ENABLED" = "yes" ]; then
echo "...disabling Bro sniffing process."
sed -i 's|BRO_ENABLED=yes|BRO_ENABLED=no|g' $CONF
fi
# stop IDS engine if running
if nsm_sensor_ps-status --only-snort-alert | grep OK >/dev/null 2>&1; then
echo "...stopping IDS sniffing process."
/usr/sbin/nsm_sensor_ps-stop --only-snort-alert >/dev/null 2>&1
fi
# disable IDS engine
if [ "$IDS_ENGINE_ENABLED" = "yes" ]; then
echo "...disabling IDS sniffing process."
sed -i 's|IDS_ENGINE_ENABLED="yes"|IDS_ENGINE_ENABLED="no"|g' $SENSORCONF
fi
# stop netsniff-ng if running
if nsm_sensor_ps-status --only-pcap | grep OK >/dev/null 2>&1; then
echo "...stopping netsniff-ng."
/usr/sbin/nsm_sensor_ps-stop --only-pcap >/dev/null 2>&1
fi
# disable netsniff-ng
if [ "$PCAP_ENABLED" = "yes" ]; then
echo "...disabling netsniff-ng."
sed -i 's|PCAP_ENABLED="yes"|PCAP_ENABLED="no"|g' $SENSORCONF
fi
# in RC2, CapMe defaults to only allowing to search back 5 years
# let's increase that to 50 years just in case
# this fix has already been applied by default in RC3
if grep " - 5 " /var/www/so/capme/.inc/callback-elastic.php >/dev/null 2>&1; then
echo "...adjusting CapMe to allow pcaps up to 50 years old."
sed -i 's| - 5 | - 50 |g' /var/www/so/capme/.inc/callback-elastic.php
fi
# generate IDS alerts and write them to standard pipeline for consumption via Sguil agents
if [ "$ENGINE" = "snort" ]; then
echo "...analyzing traffic with Snort."
snort --daq pcap -c /etc/nsm/$SENSOR/snort.conf -u sguil -g sguil -l /nsm/sensor_data/$SENSOR/snort-1 -r $PCAP >/dev/null 2>&1
else
echo "...analyzing traffic with Suricata."
suricata --user sguil --group sguil -c /etc/nsm/$SENSOR/suricata.yaml -l /nsm/sensor_data/$SENSOR --runmode single -r $PCAP >/dev/null 2>&1
fi
# generate Bro logs and write them to a unique subdirectory in /nsm/import/bro/
BRODIR="/nsm/import/bro"
echo "...analyzing traffic with Bro."
mkdir -p $BRODIR
BROLOGDIR=$(mktemp -d -p $BRODIR -t bro-XXXXXXXX)
chown $(stat $BRODIR -c %u:%g) $BROLOGDIR
chmod $(stat $BRODIR -c %a) $BROLOGDIR
cd $BROLOGDIR
/opt/bro/bin/bro -r $PCAP local >/dev/null 2>&1
cd - >/dev/null
# split traffic into daily directories so that sguil's pcap_agent can find it
if [ "$PCAP_AGENT_ENABLED" = "yes" ] ; then
START=`capinfos $PCAP -a |grep "First packet time:" | awk '{print $4}'`
END=`capinfos $PCAP -e |grep "Last packet time:" | awk '{print $4}'`
ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
CURRENT="$START"
while [ "$CURRENT" != "$ENDNEXT" ]; do
EPOCH=`date +%s --date="$CURRENT"`
PCAPDIR="/nsm/sensor_data/$SENSOR/dailylogs/$CURRENT"
FILE="$PCAPDIR/snort.log.$EPOCH"
mkdir -p $PCAPDIR
CURRENTNEXT=`date +%Y-%m-%d --date="$CURRENT 1 day"`
echo "...writing $FILE"
editcap -F libpcap -A "$CURRENT 00:00:00" -B "$CURRENTNEXT 00:00:00" $PCAP $FILE
CURRENT=`date +%Y-%m-%d --date="$CURRENT 1 day"`
done
fi
# remove temp file
rm -f $PCAP
# output final message
cat << EOF
Import complete!
You can use this hyperlink to view data in the time range of your import:
https://localhost/app/kibana#/dashboard/94b52620-342a-11e7-9d52-4f090484f59e?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START}T00:00:00.000Z',mode:absolute,to:'${ENDNEXT}T00:00:00.000Z'))
or you can manually set your Time Range to be:
From: $START To: $ENDNEXT
Please note that it may take 30 seconds or more for events to appear in Kibana.
EOF