From 8bf64362401349ee583b752e978e7895d8027cb2 Mon Sep 17 00:00:00 2001 From: cstella Date: Sat, 29 Apr 2017 01:17:28 -0400 Subject: [PATCH 1/6] Adding zeppelin notebook to run pcap queries. --- .../docker/rpm-docker/SPECS/metron.spec | 2 + .../config/zeppelin/metron/metron-pcap.json | 1 + .../src/main/scripts/pcap_zeppelin_run.sh | 58 +++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json create mode 100755 metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index b1ba637cfc..c435c6c3bf 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -288,7 +288,9 @@ This package installs the Metron PCAP files %{metron_home} %{metron_home}/bin/pcap_inspector.sh %{metron_home}/bin/pcap_query.sh %{metron_home}/bin/start_pcap_topology.sh +%{metron_home}/bin/pcap_zeppelin_run.sh %{metron_home}/flux/pcap/remote.yaml +%{metron_home}/config/zeppelin/metron/metron-pcap.json %attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json new file mode 100644 index 0000000000..e602e7d902 --- /dev/null +++ b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json @@ -0,0 +1 @@ +{"paragraphs":[{"text":"%md\n# Execute Packet Capture Queries\n\nSpecify the following to filter the packet capture query:\n* *end time* - The ending time of the query in yyyyMMdd format (e.g. 20170428)\n* *start time* - The starting time of the query in yyyyMMdd format (e.g. 20170428)\n* *query* - The [Stellar](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language) query (i.e. a Stellar expression that returns `true` or `false`) to specify the packets.\n\nThe available fields to use in the queries are as follows:\n* `ip_src_addr` - The source IP address of the packets filtered\n* `ip_src_port` - The source port of the packets filtered\n* `ip_dst_addr` - The destination IP address of the packets filtered\n* `ip_dst_port` - The destination port of the packets filtered\n* `packet` - The raw packet (for use with the `BYTEARRAY_MATCHER` function)\n\nYou can use any [Stellar function](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-core-functions)\n\n## Simple Boolean Expressions\n\nFor example:\n* `ip_dst_port == 8080` would return all packets where the destination port is `8080`\n* `ip_dst_port in [ 8080, 80 ]` would return all packets where the destination port is either `8080` or `80`\n\n## Common Network Functions on Metadata\n\nFor example:\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24')` would return all packets whose destination conforms to the CIDR `192.168.0.0/24`\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080` would return all packets matching the CIDR and whose destination port is `8080`\n \n## Filtering based on the Packet Contents\n\nWe use byteseek regular expressions to filter packets. The syntax for these is described [here](https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md).\n* `BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` in them anywhere.\n* `ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` and have a destination port of `8080`\n* `BYTEARRAY_MATCHER('ff(.){5}ff', packet)` would return all packets containing a binary regex with `0xff` followed by any 5 bytes and then `0xff`\n","dateUpdated":"2017-04-28T21:09:26+0000","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/markdown","editorHide":true},"settings":{"params":{"query":"ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\\\'/api/v1/\\\\\\'', packet)"},"forms":{}},"jobName":"paragraph_1493404426115_691558013","id":"20170428-183346_777875025","result":{"code":"SUCCESS","type":"HTML","msg":"

Execute Packet Capture Queries

\n

Specify the following to filter the packet capture query:

\n\n

The available fields to use in the queries are as follows:

\n\n

You can use any Stellar function

\n

Simple Boolean Expressions

\n

For example:

\n\n

Common Network Functions on Metadata

\n

For example:

\n\n

Filtering based on the Packet Contents

\n

We use byteseek regular expressions to filter packets. The syntax for these is described here.

\n\n"},"dateCreated":"2017-04-28T18:33:46+0000","dateStarted":"2017-04-28T21:09:25+0000","dateFinished":"2017-04-28T21:09:26+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:107","focus":true},{"text":"%sh\nexport RECORDS_PER_FILE=10000\nexport NUMBER_OF_REDUCERS=10\nexport DATE_FORMAT=\"yyyyMMdd\"\nexport PCAP_DATA_PATH=\"/apps/metron/pcap\"\n\n/usr/metron/0.4.0/bin/pcap_zeppelin_run.sh \"${query}\" \"${start time}\" \"${optional end time}\"","dateUpdated":"2017-04-29T04:57:15+0000","config":{"colWidth":12,"graph":{"mode":"table","height":164,"optionOpen":false,"keys":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1/clusters\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"values":[],"groups":[],"scatter":{"xAxis":{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1/clusters\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sh","editorHide":false},"settings":{"params":{"query":"BYTEARRAY_MATCHER('\\\\'/api/v1/clusters\\\\'', packet) && ip_dst_port == 8080","start time":"20170428","end time":""},"forms":{"optional end time":{"name":"optional end time","defaultValue":"","hidden":false},"query":{"name":"query","defaultValue":"","hidden":false},"start time":{"name":"start time","defaultValue":"","hidden":false}}},"jobName":"paragraph_1493403597045_-946711026","id":"20170428-181957_829993114","dateCreated":"2017-04-28T18:19:57+0000","dateStarted":"2017-04-29T04:57:15+0000","dateFinished":"2017-04-29T04:59:25+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:108","errorMessage":"","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493405015485_1187496580","id":"20170428-184335_1604096389","dateCreated":"2017-04-28T18:43:35+0000","status":"READY","progressUpdateIntervalMs":500,"$$hashKey":"object:109"}],"name":"metron/pcap","id":"2CG4RCECY","angularObjects":{"2CEW7JC5Y:shared_process":[],"2CGH4HQQ9:shared_process":[],"2CHTTUF65:shared_process":[],"2CHSTRTSD:shared_process":[],"2CFQRZBET:shared_process":[],"2CGSWV89A:shared_process":[]},"config":{"looknfeel":"default"},"info":{}} \ No newline at end of file diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh new file mode 100755 index 0000000000..61c67a0031 --- /dev/null +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh @@ -0,0 +1,58 @@ +#!/bin/bash +METRON_HOME=${METRON_HOME:-"/usr/metron/0.4.0"} +DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"} +USER=$(whoami) +USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER} +QUERY_HOME=${QUERY_HOME:-"$USER_HOMEDIR/queries"} +WEBHDFS_HOSTNAME=${WEBHDFS_HOSTNAME:-`hdfs getconf -confKey dfs.namenode.http-address`} +RAW_QUERY=${QUERY:-$1} +QUERY=$RAW_QUERY #$(printf '%q' $RAW_QUERY) +START_TIME=${START_TIME:-$2} +END_TIME=${END_TIME:-$3} +RECORDS_PER_FILE=${RECORDS_PER_FILE:-10000} +NUMBER_OF_REDUCERS=${NUMBER_OF_REDUCERS:-10} +PCAP_DATA_PATH=${PCAP_DATA_PATH:-/apps/metron/pcap} +if [ -z "$QUERY" ]; then + echo "You must specify a query." + exit 1 +fi +if [ -z "$START_TIME" ]; then + echo "You must specify a start time." + exit 2 +fi +if [ -z "$END_TIME" ]; then + CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" \"2>&1) + SUMMARY="Packets conforming to $RAW_QUERY starting at $START_TIME ending now" +else + CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" 2>&1) + SUMMARY="Packets conforming to $RAW_QUERY starting at $START_TIME ending at $END_TIME" +fi + +FAILED=$(echo $CMD | grep "Unable to complete query due to errors") +if [ -z "$FAILED" ];then + PATTERN=$(ls -ltr *.pcap | tail -n 1 | rev | awk '{print $1}' | rev | awk -F+ '{print $1}') + hadoop fs -mkdir -p $QUERY_HOME/$PATTERN && hadoop fs -put $PATTERN* $QUERY_HOME/$PATTERN + FAILED=$? + if [ $FAILED -eq 0 ];then + echo "%table" + echo $SUMMARY + for i in $(ls $PATTERN*.pcap);do + FILENAME=$(echo $i | sed 's/+/%2B/g') + echo "%html $i" + rm $i + done + else + echo "Unable to create $QUERY_HOME/$PATTERN" + exit 3 + fi +else + echo "%html 
FAILED JOB:"
+  echo "QUERY: $RAW_QUERY"
+  echo "START_TIME: $START_TIME"
+  echo "DATE_FORMAT: $DATE_FORMAT"
+  echo "METRON_HOME: $METRON_HOME"
+  echo "DATE_FORMAT: $DATE_FORMAT"
+  echo "Output:"
+  echo "$CMD"
+  echo "
" +fi From 0af6147a6c993ac1c4ceb41bc1bba97de0899116 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 1 May 2017 09:31:19 -0400 Subject: [PATCH 2/6] Adding license and such. --- .../src/main/scripts/pcap_zeppelin_run.sh | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh index 61c67a0031..6b078f8630 100755 --- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh @@ -1,5 +1,23 @@ #!/bin/bash -METRON_HOME=${METRON_HOME:-"/usr/metron/0.4.0"} +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +METRON_HOME=${METRON_HOME:-"/usr/metron/${project.version}"} DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"} USER=$(whoami) USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER} From 25e4dc9ecfbb49bb8c76789d024763a1a9775765 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 1 May 2017 12:03:27 -0400 Subject: [PATCH 3/6] Version isn't set properly. --- .../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh index 6b078f8630..63d8bf69d5 100755 --- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh @@ -17,7 +17,8 @@ # limitations under the License. # -METRON_HOME=${METRON_HOME:-"/usr/metron/${project.version}"} +METRON_VERSION=${project.version} +METRON_HOME=${METRON_HOME:-"/usr/metron/$METRON_VERSION"} DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"} USER=$(whoami) USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER} From 37dab9b07b3e80335be8c58c3eec25a77506d606 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 1 May 2017 13:37:45 -0400 Subject: [PATCH 4/6] Updating for better documentation. --- .../src/main/config/zeppelin/metron/metron-pcap.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json index e602e7d902..8c853a5fae 100644 --- a/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json +++ b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json @@ -1 +1 @@ -{"paragraphs":[{"text":"%md\n# Execute Packet Capture Queries\n\nSpecify the following to filter the packet capture query:\n* *end time* - The ending time of the query in yyyyMMdd format (e.g. 20170428)\n* *start time* - The starting time of the query in yyyyMMdd format (e.g. 20170428)\n* *query* - The [Stellar](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language) query (i.e. a Stellar expression that returns `true` or `false`) to specify the packets.\n\nThe available fields to use in the queries are as follows:\n* `ip_src_addr` - The source IP address of the packets filtered\n* `ip_src_port` - The source port of the packets filtered\n* `ip_dst_addr` - The destination IP address of the packets filtered\n* `ip_dst_port` - The destination port of the packets filtered\n* `packet` - The raw packet (for use with the `BYTEARRAY_MATCHER` function)\n\nYou can use any [Stellar function](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-core-functions)\n\n## Simple Boolean Expressions\n\nFor example:\n* `ip_dst_port == 8080` would return all packets where the destination port is `8080`\n* `ip_dst_port in [ 8080, 80 ]` would return all packets where the destination port is either `8080` or `80`\n\n## Common Network Functions on Metadata\n\nFor example:\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24')` would return all packets whose destination conforms to the CIDR `192.168.0.0/24`\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080` would return all packets matching the CIDR and whose destination port is `8080`\n \n## Filtering based on the Packet Contents\n\nWe use byteseek regular expressions to filter packets. The syntax for these is described [here](https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md).\n* `BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` in them anywhere.\n* `ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` and have a destination port of `8080`\n* `BYTEARRAY_MATCHER('ff(.){5}ff', packet)` would return all packets containing a binary regex with `0xff` followed by any 5 bytes and then `0xff`\n","dateUpdated":"2017-04-28T21:09:26+0000","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/markdown","editorHide":true},"settings":{"params":{"query":"ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\\\'/api/v1/\\\\\\'', packet)"},"forms":{}},"jobName":"paragraph_1493404426115_691558013","id":"20170428-183346_777875025","result":{"code":"SUCCESS","type":"HTML","msg":"

Execute Packet Capture Queries

\n

Specify the following to filter the packet capture query:

\n
    \n
  • end time - The ending time of the query in yyyyMMdd format (e.g. 20170428)
    • \n
  • start time - The starting time of the query in yyyyMMdd format (e.g. 20170428)
    • \n
  • query - The Stellar query (i.e. a Stellar expression that returns true or false) to specify the packets.
    • \n
\n

The available fields to use in the queries are as follows:

\n
    \n
  • ip_src_addr - The source IP address of the packets filtered
    • \n
  • ip_src_port - The source port of the packets filtered
    • \n
  • ip_dst_addr - The destination IP address of the packets filtered
    • \n
  • ip_dst_port - The destination port of the packets filtered
    • \n
  • packet - The raw packet (for use with the BYTEARRAY_MATCHER function)
    • \n
\n

You can use any Stellar function

\n

Simple Boolean Expressions

\n

For example:

\n
    \n
  • ip_dst_port == 8080 would return all packets where the destination port is 8080
    • \n
  • ip_dst_port in [ 8080, 80 ] would return all packets where the destination port is either 8080 or 80
    • \n
\n

Common Network Functions on Metadata

\n

For example:

\n
    \n
  • IN_SUBNET(ip_dst_addr, '192.168.0.0/24') would return all packets whose destination conforms to the CIDR 192.168.0.0/24
    • \n
  • IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080 would return all packets matching the CIDR and whose destination port is 8080
    • \n
\n

Filtering based on the Packet Contents

\n

We use byteseek regular expressions to filter packets. The syntax for these is described here.

\n
    \n
  • BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) would return all packets that contain the string /api/v1 in them anywhere.
    • \n
  • ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) would return all packets that contain the string /api/v1 and have a destination port of 8080
    • \n
  • BYTEARRAY_MATCHER('ff(.){5}ff', packet) would return all packets containing a binary regex with 0xff followed by any 5 bytes and then 0xff
    • \n
\n"},"dateCreated":"2017-04-28T18:33:46+0000","dateStarted":"2017-04-28T21:09:25+0000","dateFinished":"2017-04-28T21:09:26+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:107","focus":true},{"text":"%sh\nexport RECORDS_PER_FILE=10000\nexport NUMBER_OF_REDUCERS=10\nexport DATE_FORMAT=\"yyyyMMdd\"\nexport PCAP_DATA_PATH=\"/apps/metron/pcap\"\n\n/usr/metron/0.4.0/bin/pcap_zeppelin_run.sh \"${query}\" \"${start time}\" \"${optional end time}\"","dateUpdated":"2017-04-29T04:57:15+0000","config":{"colWidth":12,"graph":{"mode":"table","height":164,"optionOpen":false,"keys":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1/clusters\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"values":[],"groups":[],"scatter":{"xAxis":{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1/clusters\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sh","editorHide":false},"settings":{"params":{"query":"BYTEARRAY_MATCHER('\\\\'/api/v1/clusters\\\\'', packet) && ip_dst_port == 8080","start time":"20170428","end time":""},"forms":{"optional end time":{"name":"optional end time","defaultValue":"","hidden":false},"query":{"name":"query","defaultValue":"","hidden":false},"start time":{"name":"start time","defaultValue":"","hidden":false}}},"jobName":"paragraph_1493403597045_-946711026","id":"20170428-181957_829993114","dateCreated":"2017-04-28T18:19:57+0000","dateStarted":"2017-04-29T04:57:15+0000","dateFinished":"2017-04-29T04:59:25+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:108","errorMessage":"","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493405015485_1187496580","id":"20170428-184335_1604096389","dateCreated":"2017-04-28T18:43:35+0000","status":"READY","progressUpdateIntervalMs":500,"$$hashKey":"object:109"}],"name":"metron/pcap","id":"2CG4RCECY","angularObjects":{"2CEW7JC5Y:shared_process":[],"2CGH4HQQ9:shared_process":[],"2CHTTUF65:shared_process":[],"2CHSTRTSD:shared_process":[],"2CFQRZBET:shared_process":[],"2CGSWV89A:shared_process":[]},"config":{"looknfeel":"default"},"info":{}} \ No newline at end of file +{"paragraphs":[{"text":"%md\n# Execute Packet Capture Queries\n\nSpecify the following to filter the packet capture query:\n* *end time* - The ending time of the query in yyyyMMdd format (e.g. 20170428)\n* *start time* - The starting time of the query in yyyyMMdd format (e.g. 20170428)\n* *query* - The [Stellar](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language) query (i.e. a Stellar expression that returns `true` or `false`) to specify the packets.\n\nThe available fields to use in the queries are as follows:\n* `ip_src_addr` - The source IP address of the packets filtered\n* `ip_src_port` - The source port of the packets filtered\n* `ip_dst_addr` - The destination IP address of the packets filtered\n* `ip_dst_port` - The destination port of the packets filtered\n* `packet` - The raw packet (for use with the `BYTEARRAY_MATCHER` function)\n\nYou can use any [Stellar function](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-core-functions)\n\n## Simple Boolean Expressions\n\nFor example:\n* `ip_dst_port == 8080` would return all packets where the destination port is `8080`\n* `ip_dst_port in [ 8080, 80 ]` would return all packets where the destination port is either `8080` or `80`\n\n## Common Network Functions on Metadata\n\nFor example:\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24')` would return all packets whose destination conforms to the CIDR `192.168.0.0/24`\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080` would return all packets matching the CIDR and whose destination port is `8080`\n \n## Filtering based on the Packet Contents\n\nWe use byteseek regular expressions to filter packets. The syntax for these is described [here](https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md).\n* `BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` in them anywhere.\n* `ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` and have a destination port of `8080`\n* `BYTEARRAY_MATCHER('ff(.){5}ff', packet)` would return all packets containing a binary regex with `0xff` followed by any 5 bytes and then `0xff`\n\n# The Output\nThe output will be a table of links to the various parts of the packet capture files. The files will be named in temporal order.","dateUpdated":"2017-05-01T17:36:27+0000","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"query":"ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\\\'/api/v1/\\\\\\'', packet)"},"forms":{}},"jobName":"paragraph_1493658709556_1010155640","id":"20170428-183346_777875025","result":{"code":"SUCCESS","type":"HTML","msg":"

Execute Packet Capture Queries

\n

Specify the following to filter the packet capture query:

\n
    \n
  • end time - The ending time of the query in yyyyMMdd format (e.g. 20170428)
    • \n
  • start time - The starting time of the query in yyyyMMdd format (e.g. 20170428)
    • \n
  • query - The Stellar query (i.e. a Stellar expression that returns true or false) to specify the packets.
    • \n
\n

The available fields to use in the queries are as follows:

\n
    \n
  • ip_src_addr - The source IP address of the packets filtered
    • \n
  • ip_src_port - The source port of the packets filtered
    • \n
  • ip_dst_addr - The destination IP address of the packets filtered
    • \n
  • ip_dst_port - The destination port of the packets filtered
    • \n
  • packet - The raw packet (for use with the BYTEARRAY_MATCHER function)
    • \n
\n

You can use any Stellar function

\n

Simple Boolean Expressions

\n

For example:

\n
    \n
  • ip_dst_port == 8080 would return all packets where the destination port is 8080
    • \n
  • ip_dst_port in [ 8080, 80 ] would return all packets where the destination port is either 8080 or 80
    • \n
\n

Common Network Functions on Metadata

\n

For example:

\n
    \n
  • IN_SUBNET(ip_dst_addr, '192.168.0.0/24') would return all packets whose destination conforms to the CIDR 192.168.0.0/24
    • \n
  • IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080 would return all packets matching the CIDR and whose destination port is 8080
    • \n
\n

Filtering based on the Packet Contents

\n

We use byteseek regular expressions to filter packets. The syntax for these is described here.

\n
    \n
  • BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) would return all packets that contain the string /api/v1 in them anywhere.
    • \n
  • ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) would return all packets that contain the string /api/v1 and have a destination port of 8080
    • \n
  • BYTEARRAY_MATCHER('ff(.){5}ff', packet) would return all packets containing a binary regex with 0xff followed by any 5 bytes and then 0xff
    • \n
\n

The Output

\n

The output will be a table of links to the various parts of the packet capture files. The files will be named in temporal order.

\n"},"dateCreated":"2017-05-01T17:11:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1724","dateFinished":"2017-05-01T17:36:26+0000","dateStarted":"2017-05-01T17:36:26+0000","focus":true},{"text":"%sh\nexport PCAP_ZEPPELIN_RUN=$(find /usr -name pcap_zeppelin_run.sh)\nexport RECORDS_PER_FILE=10000\nexport NUMBER_OF_REDUCERS=10\nexport DATE_FORMAT=\"yyyyMMdd\"\nexport PCAP_DATA_PATH=\"/apps/metron/pcap\"\n\n$PCAP_ZEPPELIN_RUN \"${query}\" \"${start time}\" \"${optional end time}\"","dateUpdated":"2017-05-01T17:35:04+0000","config":{"colWidth":12,"editorMode":"ace/mode/sh","editorHide":false,"graph":{"mode":"table","height":164,"optionOpen":false,"keys":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"values":[],"groups":[],"scatter":{"xAxis":{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}}},"enabled":true},"settings":{"params":{"start time":"20170428","end time":"","query":"BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) && ip_dst_port == 8080","optional end time":""},"forms":{"optional end time":{"name":"optional end time","defaultValue":"","hidden":false},"query":{"name":"query","defaultValue":"","hidden":false},"start time":{"name":"start time","defaultValue":"","hidden":false}}},"jobName":"paragraph_1493658709562_1009386142","id":"20170428-181957_829993114","result":{"code":"SUCCESS","type":"TABLE","msg":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now\n%html pcap-data-20170501173442578+0000.pcap\n","comment":"","msgTable":[[{"value":"%html pcap-data-20170501173442578+0000.pcap"}]],"columnNames":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"rows":[["%html pcap-data-20170501173442578+0000.pcap"]]},"dateCreated":"2017-05-01T17:11:49+0000","dateStarted":"2017-05-01T17:35:04+0000","dateFinished":"2017-05-01T17:34:55+0000","status":"RUNNING","progressUpdateIntervalMs":500,"$$hashKey":"object:1725","focus":true},{"dateUpdated":"2017-05-01T17:34:55+0000","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/markdown","editorHide":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493658709563_1009001393","id":"20170428-184335_1604096389","dateCreated":"2017-05-01T17:11:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1726","dateFinished":"2017-05-01T17:34:54+0000","dateStarted":"2017-05-01T17:34:54+0000","result":{"code":"SUCCESS","type":"HTML","msg":"

Troubleshooting

\n

If you are having problems with the above form, the following may help.

\n

I see Terminated by SIGINTERRUPT or something similar in the output!

\n

PCAP filtering happens via a batch process and on busy systems, this can take some time. You very well may need to request this of your system administrator.\n
They can do this by changing the shell.command.timeout.millisecs property for the sh interpreter to a larger value, likely 100000.

\n

I do not see a table of URLs to pcap files in my output, what happened?

\n

If an error happens, the log of the pcap querying utility will be displayed instead of an output. Please contact an administrator with this output to debug further.

\n"},"text":"%md\n# Troubleshooting\n\nIf you are having problems with the above form, the following may help.\n\n## I see `Terminated by SIGINTERRUPT` or something similar in the output!\n\nPCAP filtering happens via a batch process and on busy systems, this can take some time. You very well may need to request this of your system administrator.\nThey can do this by changing the `shell.command.timeout.millisecs` property for the `sh` interpreter to a larger value, likely `100000`.\n\n## I do not see a table of URLs to pcap files in my output, what happened?\n\nIf an error happens, the log of the pcap querying utility will be displayed instead of an output. Please contact an administrator with this output to debug further.","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493659778398_1082698332","id":"20170501-172938_213921861","dateCreated":"2017-05-01T17:29:38+0000","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:1972"}],"name":"metron/pcap","id":"2CEEXVR1W","angularObjects":{"2CFNGE6TP:shared_process":[],"2CGUX5TSW:shared_process":[],"2CETVR2AB:shared_process":[],"2CF163WFX:shared_process":[],"2CG4YXKUV:shared_process":[],"2CG6QDFF7:shared_process":[]},"config":{"looknfeel":"default"},"info":{}} \ No newline at end of file From a0f5b1ab9e19f56e95bb7061e304d5f500500c7b Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 1 May 2017 14:24:36 -0400 Subject: [PATCH 5/6] UPdating pcap zeppelin run to not use a table. --- .../src/main/scripts/pcap_zeppelin_run.sh | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh index 63d8bf69d5..e7a4df4b97 100755 --- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh @@ -17,15 +17,14 @@ # limitations under the License. # -METRON_VERSION=${project.version} +METRON_VERSION=0.4.0 METRON_HOME=${METRON_HOME:-"/usr/metron/$METRON_VERSION"} DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"} USER=$(whoami) USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER} QUERY_HOME=${QUERY_HOME:-"$USER_HOMEDIR/queries"} WEBHDFS_HOSTNAME=${WEBHDFS_HOSTNAME:-`hdfs getconf -confKey dfs.namenode.http-address`} -RAW_QUERY=${QUERY:-$1} -QUERY=$RAW_QUERY #$(printf '%q' $RAW_QUERY) +QUERY=${QUERY:-$1} START_TIME=${START_TIME:-$2} END_TIME=${END_TIME:-$3} RECORDS_PER_FILE=${RECORDS_PER_FILE:-10000} @@ -41,10 +40,10 @@ if [ -z "$START_TIME" ]; then fi if [ -z "$END_TIME" ]; then CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" \"2>&1) - SUMMARY="Packets conforming to $RAW_QUERY starting at $START_TIME ending now" + SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending now" else CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" 2>&1) - SUMMARY="Packets conforming to $RAW_QUERY starting at $START_TIME ending at $END_TIME" + SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending at $END_TIME" fi FAILED=$(echo $CMD | grep "Unable to complete query due to errors") @@ -53,20 +52,24 @@ if [ -z "$FAILED" ];then hadoop fs -mkdir -p $QUERY_HOME/$PATTERN && hadoop fs -put $PATTERN* $QUERY_HOME/$PATTERN FAILED=$? if [ $FAILED -eq 0 ];then - echo "%table" - echo $SUMMARY - for i in $(ls $PATTERN*.pcap);do + echo "%html" + echo "

$SUMMARY

" + echo "
    " + for i in $(ls $PATTERN*.pcap | sort -n);do FILENAME=$(echo $i | sed 's/+/%2B/g') - echo "%html $i" + SIZE=$(du -h $i | awk '{print $1}') + echo "
  • $i ($SIZE)
    • " rm $i done + echo "
" + echo "NOTE: There are $RECORDS_PER_FILE records per file" else echo "Unable to create $QUERY_HOME/$PATTERN" exit 3 fi else echo "%html 
FAILED JOB:"
-  echo "QUERY: $RAW_QUERY"
+  echo "QUERY: $QUERY"
   echo "START_TIME: $START_TIME"
   echo "DATE_FORMAT: $DATE_FORMAT"
   echo "METRON_HOME: $METRON_HOME"

From 5fc6e70783ae16b6f76f2205c6b7f88bddc20863 Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 1 May 2017 15:57:24 -0400
Subject: [PATCH 6/6] Needed to add a prefix to the pcap query CLI.  This way
 we can have multiple queries dropped in the same dir.

---
 .../apache/metron/pcap/query/CliConfig.java   | 19 ++++++++++++-
 .../apache/metron/pcap/query/CliParser.java   |  4 ++-
 .../metron/pcap/query/FixedCliConfig.java     |  3 ++-
 .../metron/pcap/query/FixedCliParser.java     |  9 +++++--
 .../org/apache/metron/pcap/query/PcapCli.java | 23 ++++++++++------
 .../metron/pcap/query/QueryCliConfig.java     |  4 +++
 .../metron/pcap/query/QueryCliParser.java     |  9 +++++--
 .../src/main/scripts/pcap_zeppelin_run.sh     | 14 +++++++---
 .../apache/metron/pcap/query/PcapCliTest.java | 27 ++++++++-----------
 9 files changed, 78 insertions(+), 34 deletions(-)

diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
index 294844fe03..1d8e3f3f62 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
@@ -18,12 +18,19 @@
 package org.apache.metron.pcap.query;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.system.Clock;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
 public class CliConfig {
+  public interface PrefixStrategy extends Function{}
+
   private boolean showHelp;
+  private String prefix;
   private String basePath;
   private String baseOutputPath;
   private long startTime;
@@ -32,13 +39,23 @@ public class CliConfig {
   private int numRecordsPerFile;
   private DateFormat dateFormat;
 
-  public CliConfig() {
+
+  public CliConfig(PrefixStrategy prefixStrategy) {
     showHelp = false;
     basePath = "";
     baseOutputPath = "";
     startTime = -1L;
     endTime = -1L;
     numReducers = 0;
+    prefix = prefixStrategy.apply(new Clock());
+  }
+
+  public String getPrefix() {
+    return prefix;
+  }
+
+  public void setPrefix(String prefix) {
+    this.prefix = prefix;
   }
 
   public int getNumReducers() {
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 83e9fcfc7f..d5976ae7c9 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -29,8 +29,10 @@ public class CliParser {
   public static final int NUM_REDUCERS_DEFAULT = 10;
   public static final int NUM_RECORDS_PER_FILE_DEFAULT = 10000;
   private CommandLineParser parser;
+  protected CliConfig.PrefixStrategy prefixStrategy;
 
-  public CliParser() {
+  public CliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    this.prefixStrategy = prefixStrategy;
     parser = new PosixParser();
   }
 
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
index df653e108e..03caed7157 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
@@ -27,7 +27,8 @@ public class FixedCliConfig extends CliConfig {
 
   private Map fixedFields;
 
-  public FixedCliConfig() {
+  public FixedCliConfig(PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     this.fixedFields = new LinkedHashMap<>();
   }
 
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
index fda8692407..4e1bfcfe39 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
@@ -26,7 +26,8 @@
 public class FixedCliParser extends CliParser {
   private Options fixedOptions;
 
-  public FixedCliParser() {
+  public FixedCliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     fixedOptions = buildFixedOptions();
   }
 
@@ -38,6 +39,7 @@ private Options buildFixedOptions() {
     options.addOption(newOption("dp", "ip_dst_port", true, "Destination port"));
     options.addOption(newOption("p", "protocol", true, "IP Protocol"));
     options.addOption(newOption("pf", "packet_filter", true, "Packet Filter regex"));
+    options.addOption(newOption("pre", "prefix", true, "Result file prefix to use"));
     options.addOption(newOption("ir", "include_reverse", false, "Indicates if filter should check swapped src/dest addresses and IPs"));
     return options;
   }
@@ -51,7 +53,7 @@ private Options buildFixedOptions() {
    */
   public FixedCliConfig parse(String[] args) throws ParseException, java.text.ParseException {
     CommandLine commandLine = getParser().parse(fixedOptions, args);
-    FixedCliConfig config = new FixedCliConfig();
+    FixedCliConfig config = new FixedCliConfig(prefixStrategy);
     super.parse(commandLine, config);
     config.putFixedField(Constants.Fields.SRC_ADDR.getName(), commandLine.getOptionValue("ip_src_addr"));
     config.putFixedField(Constants.Fields.DST_ADDR.getName(), commandLine.getOptionValue("ip_dst_addr"));
@@ -60,6 +62,9 @@ public FixedCliConfig parse(String[] args) throws ParseException, java.text.Pars
     config.putFixedField(Constants.Fields.PROTOCOL.getName(), commandLine.getOptionValue("protocol"));
     config.putFixedField(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), Boolean.toString(commandLine.hasOption("include_reverse")));
     config.putFixedField(PcapHelper.PacketFields.PACKET_FILTER.getName(), commandLine.getOptionValue("packet_filter"));
+    if(commandLine.hasOption("prefix")) {
+      config.setPrefix(commandLine.getOptionValue("prefix"));
+    }
     return config;
   }
 
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index d2e6807471..b02e2e2b98 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -36,22 +36,28 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 public class PcapCli {
   private static final Logger LOGGER = LoggerFactory.getLogger(PcapCli.class);
+  public static final CliConfig.PrefixStrategy PREFIX_STRATEGY = clock -> {
+    String timestamp = new Clock().currentTimeFormatted("yyyyMMddHHmm");
+    String uuid = UUID.randomUUID().toString().replaceAll("-", "");
+    return String.format("%s-%s", timestamp, uuid);
+  };
   private final PcapJob jobRunner;
   private final ResultsWriter resultsWriter;
-  private final Clock clock;
+  private final CliConfig.PrefixStrategy prefixStrategy;
 
   public static void main(String[] args) {
-    int status = new PcapCli(new PcapJob(), new ResultsWriter(), new Clock()).run(args);
+    int status = new PcapCli(new PcapJob(), new ResultsWriter(), PREFIX_STRATEGY).run(args);
     System.exit(status);
   }
 
-  public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, Clock clock) {
+  public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, CliConfig.PrefixStrategy prefixStrategy) {
     this.jobRunner = jobRunner;
     this.resultsWriter = resultsWriter;
-    this.clock = clock;
+    this.prefixStrategy = prefixStrategy;
   }
 
   public int run(String[] args) {
@@ -72,7 +78,7 @@ public int run(String[] args) {
     }
     CliConfig commonConfig = null;
     if ("fixed".equals(jobType)) {
-      FixedCliParser fixedParser = new FixedCliParser();
+      FixedCliParser fixedParser = new FixedCliParser(prefixStrategy);
       FixedCliConfig config = null;
       try {
         config = fixedParser.parse(otherArgs);
@@ -110,7 +116,7 @@ public int run(String[] args) {
         return -1;
       }
     } else if ("query".equals(jobType)) {
-      QueryCliParser queryParser = new QueryCliParser();
+      QueryCliParser queryParser = new QueryCliParser(prefixStrategy);
       QueryCliConfig config = null;
       try {
         config = queryParser.parse(otherArgs);
@@ -151,11 +157,12 @@ public int run(String[] args) {
       return -1;
     }
     try {
+
       Iterable> partitions = Iterables.partition(results, commonConfig.getNumRecordsPerFile());
+      int part = 1;
       if (partitions.iterator().hasNext()) {
         for (List data : partitions) {
-          String timestamp = clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ");
-          String outFileName = String.format("pcap-data-%s.pcap", timestamp);
+          String outFileName = String.format("pcap-data-%s+%04d.pcap", commonConfig.getPrefix(), part++);
           if(data.size() > 0) {
             resultsWriter.write(data, outFileName);
           }
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
index 3d06e1ddc4..67f045ff8d 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
@@ -20,6 +20,10 @@
 public class QueryCliConfig extends CliConfig {
   private String query;
 
+  public QueryCliConfig(PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
+  }
+
   public String getQuery() {
     return query;
   }
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
index 72b5a95980..d6e5cd19ea 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
@@ -24,13 +24,15 @@
 public class QueryCliParser extends CliParser {
   private Options queryOptions;
 
-  public QueryCliParser() {
+  public QueryCliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     queryOptions = setupOptions();
   }
 
   private Options setupOptions() {
     Options options = buildOptions();
     options.addOption(newOption("q", "query", true, "Query string to use as a filter"));
+    options.addOption(newOption("pre", "prefix", true, "Result file prefix to use"));
     return options;
   }
 
@@ -43,11 +45,14 @@ private Options setupOptions() {
    */
   public QueryCliConfig parse(String[] args) throws ParseException, java.text.ParseException {
     CommandLine commandLine = getParser().parse(queryOptions, args);
-    QueryCliConfig config = new QueryCliConfig();
+    QueryCliConfig config = new QueryCliConfig(prefixStrategy);
     super.parse(commandLine, config);
     if (commandLine.hasOption("query")) {
       config.setQuery(commandLine.getOptionValue("query"));
     }
+    if(commandLine.hasOption("prefix")) {
+      config.setPrefix(commandLine.getOptionValue("prefix"));
+    }
     return config;
   }
 
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
index e7a4df4b97..3561c9d5c3 100755
--- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
@@ -38,17 +38,25 @@ if [ -z "$START_TIME" ]; then
   echo "You must specify a start time."
   exit 2
 fi
+TIMESTAMP_EPOCH=$(date +%s%N | cut -b1-13)
+if [ -f /proc/sys/kernel/random/uuid ];then
+  UUID=$(cat /proc/sys/kernel/random/uuid | sed 's/-//g')
+  PREFIX="$TIMESTAMP_EPOCH-$UUID"
+else
+  QUERY_HASH=$(echo $QUERY | md5sum | awk '{print $1}')
+  PREFIX="$TIMESTAMP_EPOCH-$QUERY_HASH"
+fi
 if [ -z "$END_TIME" ]; then
-  CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" \"2>&1)
+  CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" \"2>&1)
   SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending now"
 else
-  CMD=$($METRON_HOME/bin/pcap_query.sh query --query "$QUERY" -st "$START_TIME" -et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" 2>&1)
+  CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME" -et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" 2>&1)
   SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending at $END_TIME"
 fi
 
 FAILED=$(echo $CMD | grep "Unable to complete query due to errors")
 if [ -z "$FAILED" ];then
-  PATTERN=$(ls -ltr *.pcap | tail -n 1 | rev | awk '{print $1}' | rev | awk -F+ '{print $1}')
+  PATTERN="pcap-data-$PREFIX"
   hadoop fs -mkdir -p $QUERY_HOME/$PATTERN && hadoop fs -put $PATTERN* $QUERY_HOME/$PATTERN 
   FAILED=$?
   if [ $FAILED -eq 0 ];then
diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 4f441f1553..7202819f6c 100644
--- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -92,11 +92,10 @@ public void runs_fixed_pcap_filter_job_with_default_argument_list() throws Excep
     }};
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -133,11 +132,10 @@ public void runs_fixed_pcap_filter_job_with_full_argument_list_and_default_datef
     }};
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -177,11 +175,10 @@ public void runs_fixed_pcap_filter_job_with_full_argument_list() throws Exceptio
     long startAsNanos = asNanos("2016-06-13-18:35.00", "yyyy-MM-dd-HH:mm.ss");
     long endAsNanos = asNanos("2016-06-15-18:35.00", "yyyy-MM-dd-HH:mm.ss");
     when(jobRunner.query(eq(base_path), eq(base_output_path), eq(startAsNanos), eq(endAsNanos), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   private long asNanos(String inDate, String format) throws ParseException {
@@ -211,11 +208,10 @@ public void runs_query_pcap_filter_job_with_default_argument_list() throws Excep
     String query = "some query string";
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -240,11 +236,10 @@ public void runs_query_pcap_filter_job_with_full_argument_list() throws Exceptio
     String query = "some query string";
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   // INVALID OPTION CHECKS
@@ -281,7 +276,7 @@ public void assertCliError(String[] args, String type, String optMsg) {
       PrintStream errOutStream = new PrintStream(new BufferedOutputStream(ebos));
       System.setErr(errOutStream);
 
-      PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+      PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
       assertThat("Expect errors on run", cli.run(args), equalTo(-1));
       assertThat("Expect missing required option error: " + ebos.toString(), ebos.toString().contains(optMsg), equalTo(true));
       assertThat("Expect usage to be printed: " + bos.toString(), bos.toString().contains("usage: " + type + " filter options"), equalTo(true));