Permalink
Browse files

update usage statements to reflect output file option for netflow dis…

…tillation tools.
  • Loading branch information...
philhagen committed Oct 21, 2018
1 parent cec4b1a commit 6c626eb33f733df9f16b78ac077fe9024ee07149
Showing with 54 additions and 26 deletions.
  1. +4 −6 VM_README.md
  2. +15 −15 supporting-scripts/nfdump2sof-elk.sh
  3. +35 −5 supporting-scripts/vpcflow2sof-elk.sh
View
@@ -145,12 +145,10 @@ All parsers and dashboards for this VM are now maintained in this Github reposit
**Ingesting Archived NetFlow**
To ingest existing NetFlow evidence, it must be parsed into a specific format. The included nfdump2sof-elk.sh script will take care of this.
* Read from single file: ```nfdump2sof-elk.sh -r /path/to/netflow/nfcapd.201703190000```
* Read recursively from directory: ```nfdump2sof-elk.sh -r /path/to/netflow/```
* Optionally, you can specify the IP address of the exporter that created the flow data: ```nfdump2sof-elk.sh -e 10.3.58.1 -r /path/to/netflow/```
This script prints to STDOUT. Redirect to a file and place into the ```/logstash/nfarch/``` directory for parsing into SOF-ELK®.
To ingest existing NetFlow evidence, it must be parsed into a specific format. The included ```nfdump2sof-elk.sh``` script will take care of this.
* Read from single file: ```nfdump2sof-elk.sh -r /path/to/netflow/nfcapd.201703190000 -w /logstash/nfarch/inputfile_1.txt```
* Read recursively from directory: ```nfdump2sof-elk.sh -r /path/to/netflow/ -w /logstash/nfarch/inputfile_2.txt```
* Optionally, you can specify the IP address of the exporter that created the flow data: ```nfdump2sof-elk.sh -e 10.3.58.1 -r /path/to/netflow/ -w /logstash/nfarch/inputfile_3.txt```
**Sample Data**
@@ -61,12 +61,12 @@ if [[ $SOURCE_LOCATION == "" ]]; then
echoerr "Please supply a source nfcapd filename or parent directory containing nfcapd data"
echoerr " to be parsed for SOF-ELK."
echoerr ""
echoerr "Example: $0 -r /path/to/netflow/nfcapd.201703190000 -w /logstash/nfarch/<somefilename>.txt"
echoerr "Example: $0 -r /path/to/netflow/ -w /logstash/nfarch/<somefilename>.txt"
echoerr "Example: $0 -r /path/to/netflow/nfcapd.201703190000 -w /logstash/nfarch/<filename>.txt"
echoerr "Example: $0 -r /path/to/netflow/ -w /logstash/nfarch/<filename>.txt"
echoerr ""
echoerr "Can optionally supply an exporter IP address"
echoerr
echoerr "Example: $0 -e 1.2.3.4 -r /path/to/netflow/ -w /logstash/nfarch/<somefilename>.txt"
echoerr "Example: $0 -e 1.2.3.4 -r /path/to/netflow/ -w /logstash/nfarch/<filename>.txt"
echoerr ""
exit 2
fi
@@ -102,14 +102,11 @@ elif [ $MODE == "file" ]; then
READFLAG="-r"
fi
# validate source data location
nfdump $READFLAG $SOURCE_LOCATION -q -c 1 > /dev/null 2>&1
TEST_RUN=$?
if [ $TEST_RUN != 0 ]; then
# validate exporter IP address
if ! valid_ip $EXPORTER_IP; then
echoerr ""
echoerr "ERROR: Source data problem - please address prior to running this command."
exit 5
echoerr "ERROR: Invalid Exporter IP address provided - exiting."
exit 6
fi
if [ -z $DESTINATION_FILE ]; then
@@ -133,11 +130,14 @@ if [[ ! $DESTINATION_FILE =~ ^/logstash/nfarch/ ]]; then
NONSTANDARD_OUTPUT=1
fi
# validate exporter IP address
if ! valid_ip $EXPORTER_IP; then
# validate source data location
nfdump $READFLAG $SOURCE_LOCATION -q -c 1 > /dev/null 2>&1
TEST_RUN=$?
if [ $TEST_RUN != 0 ]; then
echoerr ""
echoerr "ERROR: Invalid Exporter IP address provided - exiting."
exit 6
echoerr "ERROR: Source data problem - please address prior to running this command."
exit 5
fi
# finally run nfdump command
@@ -156,6 +156,6 @@ else
echoerr " SOF-ELK can process it."
else
echoerr "SOF-ELK should now be processing the generated file - check system load and the"
echoerr " Kibana interface to confirm"
echoerr " Kibana interface to confirm."
fi
fi
@@ -21,6 +21,10 @@ while [[ $# -gt 1 ]]; do
SOURCE_LOCATION="$2"
shift # past argument
;;
-w|--destfile)
DESTINATION_FILE="$2"
shift # past argument
;;
*)
# unknown option
;;
@@ -33,8 +37,8 @@ if [[ $SOURCE_LOCATION == "" ]]; then
echoerr "Please supply a source nfcapd filename or parent directory containing VPC Flow data"
echoerr " to be parsed for SOF-ELK."
echoerr ""
echoerr "Example: $0 -r /path/to/vpcflow/dm-flowlogs.json"
echoerr "Example: $0 -r /path/to/vpcflow/"
echoerr "Example: $0 -r /path/to/vpcflow/dm-flowlogs.json -w /logstash/nfarch/<filename>.txt"
echoerr "Example: $0 -r /path/to/vpcflow/ -w /logstash/nfarch/<filename>.txt"
echoerr ""
exit 2
fi
@@ -53,6 +57,27 @@ if [ ! -d "$SOURCE_LOCATION" -a ! -f "$SOURCE_LOCATION" ]; then
exit 4
fi
if [ -z $DESTINATION_FILE ]; then
echoerr "ERROR: No destination file specified. Exiting."
exit 8
fi
if [ -d $( dirname $DESTINATION_FILE ) ]; then
DESTINATION_FILE=$( realpath $DESTINATION_FILE )
else
echoerr "ERROR: Parent path to requested destination file does not exist. Exiting."
exit 9
fi
if [[ ! $DESTINATION_FILE =~ ^/logstash/nfarch/ ]]; then
echoerr "WARNING: Output file location is not in /logstash/nfarch/. Resulting file will"
echoerr " not be automatically ingested unless moved/copied to the correct"
echoerr " filesystem location."
echoerr " Press Ctrl-C to try again or <Enter> to continue."
read
NONSTANDARD_OUTPUT=1
fi
# prepare list of input file(s) to read
# TODO: this doesn't handle spaces in directory/file names
OLDIFS=$IFS
@@ -73,7 +98,7 @@ for READFILE in $READFILES; do
fi
# finally run jq command
cat $READFILE |jq -crM '.events[].message'
cat $READFILE | jq -crM '.events[].message' > ${DESTINATION_FILE}
REAL_RUN=$?
if [ $REAL_RUN != 0 ]; then
@@ -88,8 +113,13 @@ done
echoerr ""
if [ $CONVERT_SUCCESS == 1 ]; then
echoerr "Output complete."
echoerr "If you redirected the output to a file in /logstash/nfarch/, allow SOF-ELK some time to load the data."
echoerr "If you redirected somewhere else, copy the resulting file to the /logstash/nfarch/ directory."
if [ $NONSTANDARD_OUTPUT ]; then
echoerr "You must move/copy the generated file to the /logstash/nfarch/ directory before"
echoerr " SOF-ELK can process it."
else
echoerr "SOF-ELK should now be processing the generated file - check system load and the"
echoerr " Kibana interface to confirm."
fi
else
echoerr "No files were successfully converted."
echoerr "Please validate the input data to ensure it contains proper JSON data."

0 comments on commit 6c626eb

Please sign in to comment.