From 53dcf722ee688f5854f3dc225d685fde186b0394 Mon Sep 17 00:00:00 2001 From: Mark Nottingham Date: Mon, 15 Oct 2012 13:41:35 +1100 Subject: [PATCH] initial commit --- LICENSE | 19 +++++++ README.md | 83 ++++++++++++++++++++++++++++ hdrgrab.js | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 31 +++++++++++ 4 files changed, 283 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100755 hdrgrab.js create mode 100644 package.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ca488ae --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2010-2012 Mark Nottingham + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..27b359f --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ + +# hdrgrab + +hdrgrab sniffs HTTP headers off the wire and writes them into files into +the current working directory. + +Requests and responses are written into separate files, with '-req' and +'-res' identifying each type, respectively. + +Each file contains the sets of header fields, separated by blank lines. + +Note that hdrgrab makes some effort to remove connection (hop-by-hop) headers, +but does not case normalise their names, or manipulate their values. + + +## Installing hdrgrab + +First you'll need [Node](http://nodejs.org/) and its package manager, +[npm](http://npmjs.org/). + +Then, hdrgrab can be installed with npm like this: + + > sudo npm -g install hdrgrab + +which will install dependencies automatically. + +Under the covers, hdrgrab relies upon + [node_pcap](https://github.com/mranney/node_pcap/), + [optimist](https://github.com/substack/node-optimist), and + + +## Using hdrgrab + +hdrgrab + +Start it up like this: + + > hdrgrab + +which will sniff on port 80 and dump headers into the current directory. + + > hdrgrab 8000 + +will sniff on port 80. + +On some operating systems, you may need to specify the interface to listen +on. For example: + + > hdrgrab 8000 eth0 + +and in some cases, you may need permission to listen to the device, making +the appropriate command line something like: + + > sudo hdrgrab 8000 eth0 + + + +## Installation Problems? + +### libpcap + +If npm complains about problems with pcap, like this: + + npm ERR! Failed at the pcap@0.2.7 install script. + +it usually means that it couldn't find libpcap when building. See the +instructions here: . + +On my OSX machine, I have to build like this (becoming root first): + + > CXXFLAGS=-I/opt/local/include npm -g install hdrgrab + +because my pcap headers are in a non-standard place (thanks to MacPorts). +YMMV. + + +## Contact + +Mark Nottingham + +http://github.com/mnot/hdrgrab/ + + diff --git a/hdrgrab.js b/hdrgrab.js new file mode 100755 index 0000000..36c818a --- /dev/null +++ b/hdrgrab.js @@ -0,0 +1,150 @@ +#!/usr/bin/env node + +var argv = require('optimist').argv +var dns = require('dns') +var fs = require('fs') +var node_http = require('http') +var pcap = require("pcap") +var util = require('util') + + +var hdrgrab = { + device: '', + sniff_port: 80, + captured_packets: 0, + servers: {}, + pcap_session: undefined, + drop_watcher: undefined, + err: undefined, + conn_headers: ['connection', 'keep-alive', 'te', 'transfer-encoding', 'upgrade'], + + clear: function () { + var self = this + self.packets = [] + self.capture = {sessions: {}} + self.captured_packets = 0 + self.msgs = {} + self.err = undefined + if (self.drop_watcher) { + clearInterval(self.drop_watcher) + } + self.drop_watcher = undefined + }, + + start_capture: function() { + var self = this + self.clear() + var f = "tcp port " + self.sniff_port + var b = 10 + // FIXME: where did error catch go? + self.capture.start = new Date().getTime() + self.pcap_session = pcap.createSession(self.device, f, (b * 1024 * 1024)) + this.setup_listeners() + console.log("Sniffing on " + self.pcap_session.device_name + " port " + self.sniff_port) + + // Check for pcap dropped packets on an interval + self.drop_watcher = setInterval(function () { + var stats = self.pcap_session.stats() + if (stats.ps_drop > 0) { + // TODO: notify browser through err as well + console.log( + "dropped packets, need larger buffer or less work to do: " + + util.inspect(stats) + ) + } + }, 2000) + }, + + stop_capture: function () { + var self = this + if (self.pcap_session == undefined) { + return + } + if (self.drop_watcher) { + clearInterval(self.drop_watcher) + } + self.drop_watcher == undefined + self.capture.end = new Date().getTime() + self.pcap_session.close() + self.pcap_session = undefined + console.log("Stopped sniffing") + }, + + setup_listeners: function () { + var self = this + var tcp_tracker = new pcap.TCP_tracker() + + // listen for packets, decode them, and feed TCP to the tracker + self.pcap_session.on('packet', function (raw_packet) { + self.captured_packets += 1 + var packet = pcap.decode.packet(raw_packet) + tcp_tracker.track_packet(packet) + }) + + tcp_tracker.on('http request', function (session, http) { + self.print_hdrs(http.request.headers, session, "req") + }) + + tcp_tracker.on('http response', function (session, http) { + self.print_hdrs(http.response.headers, session, "res") + }) + + tcp_tracker.on('http error', function (session, direction, error) { + console.log(" HTTP parser error: " + error) + }) + + process.on('SIGINT', function () { + console.log('Exiting.') + var server_count = 0 + for (server in self.servers) { + server_count += 1 + self.servers[server].on('close', function () { + server_count -= 1 + if (server_count == 0) { + process.exit(0) + } + }) + self.servers[server].end() + } + // if there aren't any servers + process.exit(0) + }); + }, + + print_hdrs: function (headers, session, msg_type) { + var self = this + var name = self.parse_addr(session.dst)[0] + "-" + msg_type + var server = self.servers[name] + if (! server) { + var server = fs.createWriteStream(name) + self.servers[name] = server + } + + for (line in headers) { + var lc_hdr = line.toLowerCase() + if (self.conn_headers.indexOf(lc_hdr) < 0) { + server.write(line + ": " + headers[line] + "\n") + } + } + server.write("\n") + }, + + parse_addr: function (addr) { + return addr.split(":") + } +} + + +// port to listen to +var port = parseInt(argv._[0], 10) +if (port) { + hdrgrab.sniff_port = port +} + +// device to snoop on +var device = argv._[1] +if (device) { + hdrgrab.device = device +} + +hdrgrab.start_capture() \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..13c6fdb --- /dev/null +++ b/package.json @@ -0,0 +1,31 @@ +{ "name" : "hdrgrab" +, "description" : "HTTP Header Grabber" +, "keywords" : [ "sniffer", "HTTP", "headers" ] +, "version" : "0.1.0" +, "homepage" : "https://github.com/mnot/hdrgrab" +, "author" : "Mark Nottingham (http://www.mnot.net/)" +, "contributors" : + [ + ] +, "repository" : + { "type" : "git" + , "url" : "http://github.com/mnot/hdrgrab.git" + } +, "bugs" : + { "web" : "http://github.com/mnot/hdrgrab/issues" + } +, "bin" : { "hdrgrab" : "./hdrgrab.js" } +, "engines" : + { "node" : ">=0.4.0" + , "npm" : ">=0.2.18" + } +, "dependencies" : + { "pcap" : ">=0.2.8" + , "optimist" : ">=0.1.5" + } +, "licenses" : + [ { "type" : "MIT" + , "url" : "http://github.com/mnot/hdrgrab/raw/master/LICENSE" + } + ] +} \ No newline at end of file