initial commit

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010-2012 Mark Nottingham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,83 @@
+
+# hdrgrab
+
+hdrgrab sniffs HTTP headers off the wire and writes them into files into
+the current working directory.
+
+Requests and responses are written into separate files, with '-req' and 
+'-res' identifying each type, respectively. 
+
+Each file contains the sets of header fields, separated by blank lines.
+
+Note that hdrgrab makes some effort to remove connection (hop-by-hop) headers,
+but does not case normalise their names, or manipulate their values. 
+
+
+## Installing hdrgrab
+
+First you'll need [Node](http://nodejs.org/) and its package manager, 
+[npm](http://npmjs.org/). 
+
+Then, hdrgrab can be installed with npm like this:
+
+  > sudo npm -g install hdrgrab
+
+which will install dependencies automatically. 
+
+Under the covers, hdrgrab relies upon
+ [node_pcap](https://github.com/mranney/node_pcap/), 
+ [optimist](https://github.com/substack/node-optimist), and 
+
+
+## Using hdrgrab
+
+hdrgrab
+
+Start it up like this:
+
+  > hdrgrab
+
+which will sniff on port 80 and dump headers into the current directory.
+
+  > hdrgrab 8000
+
+will sniff on port 80.
+
+On some operating systems, you may need to specify the interface to listen
+on. For example:
+
+  > hdrgrab 8000 eth0
+  
+and in some cases, you may need permission to listen to the device, making 
+the appropriate command line something like:
+
+  > sudo hdrgrab 8000 eth0
+
+
+
+## Installation Problems?
+
+### libpcap
+
+If npm complains about problems with pcap, like this:
+
+    npm ERR! Failed at the pcap@0.2.7 install script.
+
+it usually means that it couldn't find libpcap when building. See the
+instructions here: <https://github.com/mranney/node_pcap>. 
+
+On my OSX machine, I have to build like this (becoming root first):
+
+  > CXXFLAGS=-I/opt/local/include npm -g install hdrgrab
+  
+because my pcap headers are in a non-standard place (thanks to MacPorts). 
+YMMV.
+
+
+## Contact
+
+Mark Nottingham <mnot@mnot.net>
+
+http://github.com/mnot/hdrgrab/
+
+

hdrgrab.js

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+
+var argv = require('optimist').argv
+var dns = require('dns')
+var fs = require('fs')
+var node_http = require('http')
+var pcap = require("pcap")
+var util = require('util')
+
+
+var hdrgrab = {
+  device: '',
+  sniff_port: 80,
+  captured_packets: 0,
+  servers: {},
+  pcap_session: undefined,
+  drop_watcher: undefined,
+  err: undefined,
+  conn_headers: ['connection', 'keep-alive', 'te', 'transfer-encoding', 'upgrade'],
+
+  clear: function () {
+    var self = this
+    self.packets = []
+    self.capture = {sessions: {}}
+    self.captured_packets = 0
+    self.msgs = {}
+    self.err = undefined
+    if (self.drop_watcher) {
+      clearInterval(self.drop_watcher)
+    }
+    self.drop_watcher = undefined
+  },
+
+  start_capture: function() {
+    var self = this
+    self.clear()
+    var f = "tcp port " + self.sniff_port
+    var b = 10
+    // FIXME: where did error catch go?
+    self.capture.start = new Date().getTime()
+    self.pcap_session = pcap.createSession(self.device, f, (b * 1024 * 1024))
+    this.setup_listeners()
+    console.log("Sniffing on " + self.pcap_session.device_name + " port " + self.sniff_port)
+
+    // Check for pcap dropped packets on an interval
+    self.drop_watcher = setInterval(function () {
+      var stats = self.pcap_session.stats()
+      if (stats.ps_drop > 0) {
+        // TODO: notify browser through err as well
+        console.log(
+          "dropped packets, need larger buffer or less work to do: " 
+          + util.inspect(stats)
+        )
+      }
+    }, 2000)
+  },
+
+  stop_capture: function () {
+    var self = this
+    if (self.pcap_session == undefined) {
+      return
+    }
+    if (self.drop_watcher) {
+      clearInterval(self.drop_watcher)
+    }
+    self.drop_watcher == undefined
+    self.capture.end = new Date().getTime()
+    self.pcap_session.close()
+    self.pcap_session = undefined
+    console.log("Stopped sniffing")
+  },
+
+  setup_listeners: function () {
+    var self = this
+    var tcp_tracker = new pcap.TCP_tracker()
+
+    // listen for packets, decode them, and feed TCP to the tracker
+    self.pcap_session.on('packet', function (raw_packet) {
+      self.captured_packets += 1
+      var packet = pcap.decode.packet(raw_packet)
+      tcp_tracker.track_packet(packet)
+    })
+
+    tcp_tracker.on('http request', function (session, http) {
+      self.print_hdrs(http.request.headers, session, "req")
+    })
+
+    tcp_tracker.on('http response', function (session, http) {
+      self.print_hdrs(http.response.headers, session, "res")
+    })
+
+    tcp_tracker.on('http error', function (session, direction, error) {
+      console.log(" HTTP parser error: " + error)
+    })
+
+    process.on('SIGINT', function () {
+      console.log('Exiting.')
+      var server_count = 0
+      for (server in self.servers) {
+        server_count += 1
+        self.servers[server].on('close', function () {
+          server_count -= 1
+          if (server_count == 0) {
+            process.exit(0)
+          }
+        })
+        self.servers[server].end()
+      }
+      // if there aren't any servers
+      process.exit(0)
+    });
+  },
+
+  print_hdrs: function (headers, session, msg_type) {
+    var self = this
+    var name = self.parse_addr(session.dst)[0] + "-" + msg_type
+    var server = self.servers[name]
+    if (! server) {
+      var server = fs.createWriteStream(name)
+      self.servers[name] = server
+    }
+
+    for (line in headers) {
+      var lc_hdr = line.toLowerCase()
+      if (self.conn_headers.indexOf(lc_hdr) < 0) {
+        server.write(line + ": " + headers[line] + "\n")
+      }
+    }
+    server.write("\n")
+  }, 
+
+  parse_addr: function (addr) {
+    return addr.split(":")
+  }
+}
+
+
+// port to listen to 
+var port = parseInt(argv._[0], 10)
+if (port) {
+  hdrgrab.sniff_port = port
+}
+
+// device to snoop on
+var device = argv._[1]
+if (device) {
+  hdrgrab.device = device
+}
+
+hdrgrab.start_capture()

package.json

@@ -0,0 +1,31 @@
+{ "name" : "hdrgrab"
+, "description" : "HTTP Header Grabber"
+, "keywords" : [ "sniffer", "HTTP", "headers" ]
+, "version" : "0.1.0"
+, "homepage" : "https://github.com/mnot/hdrgrab"
+, "author" : "Mark Nottingham <mnot@mnot.net> (http://www.mnot.net/)"
+, "contributors" :
+  [ 
+  ]
+, "repository" :
+  { "type" : "git"
+  , "url" : "http://github.com/mnot/hdrgrab.git"
+  }
+, "bugs" :
+  { "web" : "http://github.com/mnot/hdrgrab/issues"
+  }
+, "bin" : { "hdrgrab" : "./hdrgrab.js" }
+, "engines" : 
+  { "node" : ">=0.4.0"
+  , "npm" : ">=0.2.18"
+  }
+, "dependencies" : 
+  { "pcap" : ">=0.2.8"
+  , "optimist" : ">=0.1.5"
+  }
+, "licenses" :
+  [ { "type" : "MIT"
+    , "url" : "http://github.com/mnot/hdrgrab/raw/master/LICENSE"
+    }
+  ]
+}