Improved cookie support and NTLM autentication.

Gemfile

@@ -2,13 +2,14 @@ source "http://rubygems.org"
 
 gem "i18n"
 gem "eventmachine", "1.0.0.beta.3"
-gem "em-http-request"
+gem "em-http-request", "1.0.0.beta.3"
 gem "cookiejar"
 gem "ruby-ntlm"
 gem "thor"
 gem "json"
 gem "activesupport", "~> 3.0", :path => "~/Code/rails" #:git => "https://github.com/rails/rails.git"
 gem "rainbow"
+gem "macaddr"
 
 group :development do
   gem "bundler", "~> 1.0.0"

Gemfile.lock

@@ -8,19 +8,23 @@ GEM
   specs:
     addressable (2.2.4)
     cookiejar (0.3.0)
-    em-http-request (0.3.0)
-      addressable (>= 2.0.0)
-      escape_utils
-      eventmachine (>= 0.12.9)
-    escape_utils (0.2.3)
+    em-http-request (1.0.0.beta.3)
+      addressable (>= 2.2.3)
+      em-socksify
+      eventmachine
+      http_parser.rb (>= 0.5.1)
+    em-socksify (0.1.0)
+      eventmachine
     eventmachine (1.0.0.beta.3)
     git (1.2.5)
+    http_parser.rb (0.5.1)
     i18n (0.5.0)
     jeweler (1.5.2)
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
     json (1.5.1)
+    macaddr (1.0.0)
     rainbow (1.1.1)
     rake (0.8.7)
     rr (1.0.2)
@@ -36,11 +40,12 @@ DEPENDENCIES
   activesupport (~> 3.0)!
   bundler (~> 1.0.0)
   cookiejar
-  em-http-request
+  em-http-request (= 1.0.0.beta.3)
   eventmachine (= 1.0.0.beta.3)
   i18n
   jeweler (~> 1.5.2)
   json
+  macaddr
   rainbow
   rr
   ruby-ntlm

lib/stampede/modules/http.rb

@@ -1,10 +1,13 @@
 require "em-http-request"
 require "cookiejar"
+require "ntlm"
 
 module Stampede
   module Modules::HTTP
     METHODS = [:get, :post, :put, :delete, :head]
-    HEADERS = { "user-agent" => Stampede.user_agent }
+
+    DEFAULT_OPTIONS = { :keepalive => true, :redirects => 0, :inactivity_timeout => 30 }
+    DEFAULT_HEADERS = { "user-agent" => Stampede.user_agent }
 
     METHODS.each do |method|
       class_eval <<-RUBY
@@ -14,6 +17,41 @@ def #{method}(url, options = {})
       RUBY
     end
 
+    def authenticate(username, password)
+      domain = username.slice!(/\A\w+[\\\/]/)
+      if domain and domain.chop!
+        # NTLM
+        options :ntlm_auth => [username, domain, password]
+        headers "authorization" => "NTLM " + NTLM.negotiate.to_base64
+      else
+        # HTTP Basic
+        headers "authorization" => [username, password]
+      end
+    end
+
+    def user_agent(agent)
+      headers "user-agent" => agent
+    end
+
+    def headers(head)
+      extend_parent
+      self.http_headers.merge! head
+    end
+
+    def options(options)
+      extend_parent
+      self.http_options.merge! options
+    end
+
+    private
+
+    def extend_parent
+      return if respond_to? :http_headers and respond_to? :http_options
+      class_attribute :http_headers, :http_options
+      self.http_headers = {}
+      self.http_options = {}
+    end
+
     class Request < Action
       class_attribute :http_method, :url, :options
 
@@ -25,68 +63,110 @@ def initialize(http_method, url, options = {})
       end
 
       def start
-        request = build_request
-        report :method => http_method
+        @requests = 0
+        start_request http_method, url, collect_options
+      end
+
+      def finish_request
+        @requests -= 1
+        finish if @requests == 0
+      end
+
+      def start_request(http_method, url, options)
+        @requests += 1
+
+        request = connection.send(http_method, options)
 
+        request_report = {}
         inflated_content_length = 0
         latency = nil
+        primary_request = true
 
         request.headers do
           latency ||= elapsed
           last_url = request.last_effective_url.normalize.to_s
-          set_cookies last_url, request.response_header if stateful?
-          report :head => request.response_header
-
-          report :url => last_url,
+          if stateful?
+            set_cookies last_url, request.response_header
+            if authenticate_ntlm request.response_header
+              primary_request = false
+            end
+          end
+
+          request_report.merge! :method => http_method,
+            :url => last_url,
             :status => request.response_header.status,
             :latency => latency,
             :compressed => request.response_header.compressed?
         end
 
         request.stream do |data|
           inflated_content_length += data.length
-          report_sequence :chunks,
-            :length => data.length,
-            :elapsed => elapsed
+          request_report[:chunks] ||= []
+          request_report[:chunks] << { :length => data.length, :elapsed => elapsed }
         end
 
         request.callback do
-          report :success => true,
-            :redirects => request.redirects,
-            :length => inflated_content_length || request.response.length
-          finish
+          request_report.merge!(:success => true, :length => inflated_content_length || request.response.length)
+          if primary_request
+            report request_report
+          else
+            report_sequence :subrequests, request_report
+          end
+          finish_request
         end
 
         request.errback do
-          report :success => false,
-            :error => request.error
-          finish
+          request_report.merge!(:success => false, :error => request.error)
+          if primary_request
+            report request_report
+          else
+            report_sequence :subrequests, request_report
+          end
+          finish_request
         end
       end
 
       private
 
-      def set_cookies(url, headers)
-        headers.cookie.each do |header|
-          cookiejar.set_cookie url, header
+      def set_cookies(url, header)
+        # Split the cookie header, because em-http-request (incorrectly) folds
+        # multiple Set-Cookie headers into one.
+        header.cookie.to_s.split(/(?<!expires=\w{3}),\s*/i).each do |cookie_header|
+          cookiejar.set_cookie url, cookie_header rescue nil
+        end
+      end
+
+      def authenticate_ntlm(header)
+        challenge = header["WWW_AUTHENTICATE"][/NTLM (.*)/, 1].unpack('m').first rescue nil
+        if challenge and header.status == 401
+          ntlm_response = "NTLM " + NTLM.authenticate(challenge, *@context.http_options[:ntlm_auth]).to_base64
+          start_request http_method, url, collect_options.tap { |opts| opts[:head]["authorization"] = ntlm_response }
+          true
         end
       end
 
       def cookiejar
         @cookiejar ||= (@context[:http_cookiejar] ||= CookieJar::Jar.new)
       end
 
-      def build_request
-        EM::HttpRequest.new(url).send(http_method, collect_options)
+      def connection
+        @connection ||= EM::HttpRequest.new(url)
       end
 
       def collect_options
-        options.merge :redirects => 5, :head => collect_headers, :inactivity_timeout => 30
+        DEFAULT_OPTIONS.dup.tap do |options|
+          options.merge! @context.http_options if @context.respond_to? :http_options
+          options.merge! :head => collect_headers
+        end
       end
 
       def collect_headers
-        HEADERS.dup.tap do |headers|
-          headers.merge! "cookie" => cookiejar.get_cookie_header(url) if stateful?
+        DEFAULT_HEADERS.dup.tap do |headers|
+          headers.merge! @context.http_headers if @context.respond_to? :http_headers
+          if stateful?
+            cookies = cookiejar.get_cookie_header(url)
+            headers.merge! "cookie" => cookies unless cookies.blank?
+          end
         end
       end
     end

lib/stampede/primitives/process/reporting.rb

@@ -1,4 +1,5 @@
 require "active_support/concern"
+require "macaddr"
 
 module Stampede
   # Report data to the current context.

lib/stampede/runner.rb

@@ -50,7 +50,7 @@ def abort(exit_message = "Aborted.")
     end
 
     def record(data)
-      @reporter.record(data)
+      @reporter.record({ :host => Mac.address }.merge(data))
     end
 
     private

lib/stampede/server.rb

@@ -3,6 +3,7 @@
 module Stampede
   class Server < Thor
     class_option "no-colors", :type => :boolean, :desc => "Avoid ANSI colors in output."
+    class_option "trace", :type => :boolean, :desc => "Show stack traces on errors."
 
     desc "start SCENARIO", "Starts scenario defined in the given file"
     long_desc <<-DOC
@@ -24,6 +25,7 @@ def start(scenario)
       Runner.start Scenario.from_file(scenario), options
     rescue Exception => e
       $stderr.puts e
+      $stderr.puts e.backtrace if options[:trace]
     end
 
     desc "quickstart", "Quick start guide", :hide => true

test/fixtures/http_responses/facebook.com

@@ -0,0 +1,10 @@
+HTTP/1.1 302 Found
+Location: http://m.facebook.com/?w2m&refsrc=http%3A%2F%2Fwww.facebook.com%2F&_rdr
+P3P: CP="Facebook does not have a P3P policy. Learn why here: http://fb.me/p3p"
+Set-Cookie: datr=NyOGTWIxclv1czGUC7NyUt-A; expires=Tue, 19-Mar-2013 15:54:31 GMT; path=/; domain=localhost.local; httponly
+Content-Type: text/html; charset=utf-8
+X-FB-Server: 10.52.74.37
+X-Cnection: close
+Date: Sun, 20 Mar 2011 15:54:31 GMT
+Content-Length: 0
+

test/fixtures/http_responses/facebook.com_invalid_cookie

@@ -0,0 +1,10 @@
+HTTP/1.1 302 Found
+Location: http://m.facebook.com/?w2m&refsrc=http%3A%2F%2Fwww.facebook.com%2F&_rdr
+P3P: CP="Facebook does not have a P3P policy. Learn why here: http://fb.me/p3p"
+Set-Cookie: datr=EySGTfDBIWEJuJQFJaOVPvpw; expires=Tue, 19-Mar-2013 15:58:11 GMT; path=/; domain=.facebook.com; httponly
+Content-Type: text/html; charset=utf-8
+X-FB-Server: 10.136.189.115
+X-Cnection: close
+Date: Sun, 20 Mar 2011 15:58:11 GMT
+Content-Length: 0
+