snort comparison

lib/packetpig/src/main/java/com/packetloop/packetpig/loaders/pcap/detection/SnortRecordReader.java

@@ -97,7 +97,7 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
                 throw new IOException();
             }
 
-            String[] sig_a = { m.group(2), m.group(3), m.group(4) };
+            String[] sig_a = { m.group(2), m.group(3) };
             String sig = StringUtils.join(sig_a, "_");
             String message = m.group(5);
             int priority = Integer.parseInt(m.group(6));

lib/packetpig/src/main/java/com/packetloop/packetpig/loaders/pcap/protocol/HTTPConversationLoader.java

@@ -18,7 +18,7 @@ public HTTPConversationLoader(String field) {
         this.field = field;
     }
 
-    public HTTPConversationLoader(String pathToTcp, String field) {
+    public HTTPConversationLoader(String field, String pathToTcp) {
         this.pathToTcp = pathToTcp;
         this.field = field;
     }

pig/examples/http.pig

@@ -3,6 +3,7 @@ RUN $includepath;
 
 %DEFAULT time 60
 %DEFAULT field ''
+%DEFAULT tcppath 'lib/scripts/tcp.py'
 
 http = LOAD '$pcap' USING com.packetloop.packetpig.loaders.pcap.protocol.HTTPConversationLoader('$field', '$tcppath') AS (
     ts:long,

pig/examples/snort_comparison.pig

@@ -0,0 +1,77 @@
+%DEFAULT includepath pig/include.pig
+RUN $includepath;
+
+%DEFAULT time 60
+
+snort_2905_alerts = 
+  LOAD '/Users/david/Downloads/captures/aurora_meterpreter_http.cap'
+  USING com.packetloop.packetpig.loaders.pcap.detection.SnortLoader('lib/snort-2905/etc/snort.conf')
+  AS (
+    ts:long,
+    sig:chararray,
+    priority:int,
+    message:chararray,
+    proto:chararray,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int
+  );
+
+snort_2931_alerts = 
+  LOAD '/Users/david/Downloads/captures/aurora_meterpreter_http.cap'
+  USING com.packetloop.packetpig.loaders.pcap.detection.SnortLoader('lib/snort-2931/etc/snort.conf')
+  AS (
+    ts:long,
+    sig:chararray,
+    priority:int,
+    message:chararray,
+    proto:chararray,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int
+  );
+
+-- snort_2905_alerts = 
+-- LOAD 'snort_2905' AS (
+--     ts:long,
+--     sig:chararray,
+--     severity:int,
+--     message:chararray,
+--     proto:chararray,
+--     src:chararray,
+--     sport:int,
+--     dst:chararray,
+--     dport:int
+-- );
+-- 
+-- snort_2931_alerts = 
+-- LOAD 'snort_2931' AS (
+--     ts:long,
+--     sig:chararray,
+--     severity:int,
+--     message:chararray,
+--     proto:chararray,
+--     src:chararray,
+--     sport:int,
+--     dst:chararray,
+--     dport:int
+-- );
+
+snort_2905_sigs = FOREACH snort_2905_alerts GENERATE sig, message;
+snort_2931_sigs = FOREACH snort_2931_alerts GENERATE sig, message;
+
+snort_2905_grouped = GROUP snort_2905_sigs BY sig;
+snort_2931_grouped = GROUP snort_2931_sigs BY sig;
+
+snort_2905_summed = FOREACH snort_2905_grouped GENERATE group, COUNT(snort_2905_sigs);
+snort_2931_summed = FOREACH snort_2931_grouped GENERATE group, COUNT(snort_2931_sigs);
+
+snort_summed_joined = COGROUP snort_2905_summed BY group,
+                              snort_2931_summed BY group;
+
+new_only_filtered = FILTER snort_summed_joined BY (COUNT(snort_2905_summed) == 0);
+new_only_flattened = FOREACH new_only_filtered GENERATE FLATTEN(snort_2931_summed);
+
+DUMP new_only_flattened;

pig/examples/user_agent.pig

@@ -0,0 +1,19 @@
+%DEFAULT includepath pig/include.pig
+RUN $includepath;
+
+%DEFAULT time 60
+%DEFAULT field ''
+%DEFAULT tcppath 'lib/scripts/tcp.py'
+
+http = LOAD '$pcap' USING com.packetloop.packetpig.loaders.pcap.protocol.HTTPConversationLoader('user-agent', '$tcppath') AS (
+    ts:long,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int,
+    request:chararray,
+    fields:tuple()
+);
+
+STORE http INTO '$output/http';
+

pig/examples/user_info.pig

@@ -0,0 +1,53 @@
+%DEFAULT includepath pig/include.pig
+RUN $includepath;
+
+%DEFAULT time 60
+%DEFAULT field ''
+%DEFAULT tcppath 'lib/scripts/tcp.py'
+%DEFAULT snortconfig 'lib/snort/etc/snort.conf'
+
+http = LOAD '/pl/dumps/174f501a-d6e3-11e1-bec6-7f3db9d5953d' USING com.packetloop.packetpig.loaders.pcap.protocol.HTTPConversationLoader('user-agent') AS (
+    ts:long,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int,
+    request:chararray,
+    fields:tuple()
+);
+
+snort_alerts = LOAD '/pl/dumps/174f501a-d6e3-11e1-bec6-7f3db9d5953d' USING com.packetloop.packetpig.loaders.pcap.detection.SnortLoader() AS (
+    ts:long,
+    sig:chararray,
+    priority:int,
+    message:chararray,
+    proto:chararray,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int
+);
+
+fingerprints = LOAD '/pl/dumps/174f501a-d6e3-11e1-bec6-7f3db9d5953d' USING com.packetloop.packetpig.loaders.pcap.detection.FingerprintLoader() AS (
+    ts:long,
+    src:chararray,
+    sport:int,
+    dst:chararray,
+    dport:int,
+    os:chararray
+);
+
+attacker_fingerprint_info = JOIN
+                            snort_alerts BY (src, sport, dst, dport),
+                            fingerprints BY (src, sport, dst, dport);
+
+attacker_fingerprints = FOREACH attacker_fingerprint_info GENERATE kkkkkkkkkkkkkkkkk
+
+dump attacker_fingerprints;
+
+--attacker_useragents = JOIN
+--                        attacker_fingerprints BY (src, sport, dst, dport),
+--                        http BY (src, sport, dst, dport);
+--
+--STORE attacker_useragents INTO '$output/user_info';
+

pigrun.py

@@ -31,7 +31,6 @@ def generate_cmd(conf):
     cmd = []
     cmd.append('pig -v')
     cmd.append('-x ' + conf.mode)
-    cmd.append('-param output=output')
 
     if conf.mode == 'mapreduce':
         pig_path = prepend_hdfs_path(conf, conf.pig_path)
@@ -40,16 +39,17 @@ def generate_cmd(conf):
         cmd.append('-f %s' % pig_path)
         cmd.append('-param pcap=%s' % pcap_path)
         cmd.append('-param includepath=%s/include-hdfs.pig' % conf.hdfs_path)
-        #cmd.append('-param cvss=%s/snort-cvss.tsv' % conf.hdfs_path)
-        cmd.append('-param tcppath=%s' % conf.tcp_path)
-        cmd.append('-param dnspath=%s' % conf.dns_path)
 
     if conf.mode == 'local':
         cmd.append('-f %s' % conf.pig_path)
         cmd.append('-param pcap=%s' % conf.pcap_path)
 
+    cmd.append('-param output=output')
     cmd.append('-param n=%s' % conf.n)
     cmd.append('-param snortconfig=%s' % conf.snort_conf)
+    #cmd.append('-param cvss=%s/snort-cvss.tsv' % conf.hdfs_path)
+    cmd.append('-param tcppath=%s' % conf.tcp_path)
+    cmd.append('-param dnspath=%s' % conf.dns_path)
 
     pprint(cmd)
     print