VCR-like cache

Gemfile.lock

@@ -13,22 +13,23 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.2)
-    capybara (1.1.2)
+    addressable (2.3.3)
+    capybara (2.0.2)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       selenium-webdriver (~> 2.0)
-      xpath (~> 0.1.4)
-    capybara-webkit (0.12.1)
-      capybara (>= 1.0.0, < 1.2)
+      xpath (~> 1.0.0)
+    capybara-webkit (0.14.2)
+      capybara (~> 2.0, >= 2.0.2)
       json
-    childprocess (0.3.5)
-      ffi (~> 1.0, >= 1.0.6)
+    childprocess (0.3.8)
+      ffi (~> 1.0, >= 1.0.11)
+    coderay (1.0.9)
     cookiejar (0.3.0)
     daemons (1.1.9)
-    diff-lcs (1.1.3)
+    diff-lcs (1.2.1)
     em-http-request (1.0.3)
       addressable (>= 2.2.3)
       cookiejar
@@ -39,47 +40,62 @@ GEM
       eventmachine (>= 1.0.0.beta.4)
     eventmachine (1.0.0)
     eventmachine_httpserver (0.2.1)
-    faraday (0.8.4)
+    faraday (0.8.6)
       multipart-post (~> 1.1)
-    faye-websocket (0.4.6)
+    faye-websocket (0.4.7)
       eventmachine (>= 0.12.0)
-    ffi (1.1.5)
+    ffi (1.4.0)
+    guard (1.6.2)
+      listen (>= 0.6.0)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      terminal-table (>= 1.4.3)
+      thor (>= 0.14.6)
     http_parser.rb (0.5.3)
-    json (1.7.5)
-    libwebsocket (0.1.5)
-      addressable
-    mime-types (1.19)
-    multi_json (1.3.6)
-    multipart-post (1.1.5)
-    nokogiri (1.5.5)
-    poltergeist (0.7.0)
-      capybara (~> 1.1)
-      childprocess (~> 0.3)
+    json (1.7.7)
+    listen (0.7.3)
+    lumberjack (1.0.2)
+    method_source (0.8.1)
+    mime-types (1.21)
+    multi_json (1.6.1)
+    multipart-post (1.2.0)
+    nokogiri (1.5.6)
+    poltergeist (1.1.0)
+      capybara (~> 2.0, >= 2.0.1)
       faye-websocket (~> 0.4, >= 0.4.4)
       http_parser.rb (~> 0.5.3)
-      multi_json (~> 1.0)
-    rack (1.4.1)
+    pry (0.9.12)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
     rack-test (0.6.2)
       rack (>= 1.0)
-    rspec (2.11.0)
-      rspec-core (~> 2.11.0)
-      rspec-expectations (~> 2.11.0)
-      rspec-mocks (~> 2.11.0)
-    rspec-core (2.11.1)
-    rspec-expectations (2.11.3)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.3)
+    rb-inotify (0.9.0)
+      ffi (>= 0.5.0)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.0)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.0)
     rubyzip (0.9.9)
-    selenium-webdriver (2.25.0)
+    selenium-webdriver (2.30.0)
       childprocess (>= 0.2.5)
-      libwebsocket (~> 0.1.3)
       multi_json (~> 1.0)
       rubyzip
-    thin (1.4.1)
+      websocket (~> 1.0.4)
+    slop (3.4.3)
+    terminal-table (1.4.5)
+    thin (1.5.0)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
-    xpath (0.1.4)
+    thor (0.17.0)
+    websocket (1.0.7)
+    xpath (1.0.0)
       nokogiri (~> 1.3)
     yajl-ruby (1.1.0)
 
@@ -89,9 +105,11 @@ PLATFORMS
 DEPENDENCIES
   capybara-webkit
   faraday
+  guard
   poltergeist
   puffing-billy!
   rack
+  rb-inotify
   rspec
   selenium-webdriver
   thin

Guardfile

@@ -0,0 +1,23 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+
+  # Capybara request specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/requests/#{m[1]}_spec.rb" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end

README.md

@@ -85,6 +85,42 @@ proxy.stub('https://example.com/proc/').and_return(Proc.new { |params, headers,
 Stubs are reset between tests.  Any requests that are not stubbed will be
 proxied to the remote server.
 
+## Caching
+
+Requests routed through the external proxy are cached.
+
+If you want to use puffing-billy like you would [VCR](https://github.com/vcr/vcr) you can turn on cache persistence. 
+This way you don't have to manually mock out everything as requests are automatically recorded and played back.
+With cache persistence you can take tests completely offline.
+
+In your `spec_helper.rb`:
+
+```ruby
+Billy.configure do |c|
+  c.cache = true
+  c.ignore_params = ["http://www.google-analytics.com/__utm.gif",
+                     "https://r.twimg.com/jot",
+                     "http://p.twitter.com/t.gif",
+                     "http://p.twitter.com/f.gif",
+                     "http://www.facebook.com/plugins/like.php",
+                     "https://www.facebook.com/dialog/oauth",
+                     "http://cdn.api.twitter.com/1/urls/count.json"]
+  c.persist_cache = true
+  c.cache_path = 'spec/req_cache/'
+end
+
+# need to call this because of a race condition between persist_cache
+# being set and the proxy being loaded for the first time
+Billy.proxy.restore_cache
+```
+
+`c.ignore_params` is used to ignore parameters of certain requests when caching. You should mostly use this for analytics
+and various social buttons as they use cache avoidance techniques, but return practically the same response that most often 
+does not affect your test results.
+
+The cache works with all types of requests and will distinguish between differend POST requests to the same URL.
+
+
 ## Customising the javascript driver
 
 If you use a customised Capybara driver, remember to set the proxy address

lib/billy/cache.rb

@@ -1,10 +1,12 @@
 require 'resolv'
 require 'uri'
+require 'yaml'
 
 module Billy
   class Cache
     def initialize
       reset
+      load_dir
     end
 
     def cacheable?(url, headers)
@@ -16,24 +18,71 @@ def cacheable?(url, headers)
       end
     end
 
-    def cached?(url)
-      !@cache[url].nil?
+    def cached?(method, url, body)
+      !@cache[key(method, url, body)].nil?
     end
 
-    def fetch(url)
-      @cache[url]
+    def fetch(method, url, body)
+      @cache[key(method, url, body)]
     end
 
-    def store(url, status, headers, content)
-      @cache[url] = {
+    def store(method, url, body, status, headers, content)
+      cached = {
+        :url => url,
+        :body => body,
         :status => status,
+        :method => method,
         :headers => headers,
         :content => content
       }
+
+      @cache[key(method, url, body)] = cached
+
+      if Billy.config.persist_cache
+        Dir.mkdir(Billy.config.cache_path) unless File.exists?(Billy.config.cache_path)
+
+        begin
+          File.open(Billy.config.cache_path+key(method, url, body)+".yml", 'w') {
+            |f| f.write(cached.to_yaml(:Encoding => :Utf8))
+          }
+        rescue StandardError => e
+        end
+      end
     end
 
     def reset
       @cache = {}
     end
+
+    def load_dir
+      if Billy.config.persist_cache
+        Dir.glob(Billy.config.cache_path+"*.yml") { |filename|
+          data = begin
+                   YAML.load(File.open(filename))
+                 rescue ArgumentError => e
+                   puts "Could not parse YAML: #{e.message}"
+                 end
+
+          @cache[key(data[:method], data[:url], data[:body])] = data
+        }
+      end
+    end
+
+    def key(method, url, body)
+      url = URI(url)
+      no_params = url.scheme+'://'+url.host+url.path
+
+      if Billy.config.ignore_params.include?(no_params)
+        url = URI(no_params)
+      end
+
+      key = method+'_'+url.host+'_'+Digest::SHA1.hexdigest(url.to_s)
+
+      if method == 'post' and !Billy.config.ignore_params.include?(no_params)
+        key += '_'+Digest::SHA1.hexdigest(body.to_s)
+      end
+
+      key
+    end
   end
 end

lib/billy/config.rb

@@ -4,12 +4,15 @@ module Billy
   class Config
     DEFAULT_WHITELIST = ['127.0.0.1', 'localhost']
 
-    attr_accessor :logger, :cache, :whitelist
+    attr_accessor :logger, :cache, :whitelist, :ignore_params, :persist_cache, :cache_path
 
     def initialize
       @logger    = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
       @cache     = true
       @whitelist = DEFAULT_WHITELIST
+      @ignore_params = []
+      @persist_cache = false
+      @cache_path = '/tmp'
     end
   end
 

lib/billy/proxy.rb

@@ -53,6 +53,11 @@ def reset_cache
       @cache.reset
     end
 
+    def restore_cache
+      @cache.reset
+      @cache.load_dir
+    end
+
     protected
 
     def find_stub(method, url)

lib/billy/proxy_connection.rb

@@ -72,14 +72,15 @@ def handle_request
       if handler && handler.respond_to?(:call)
         result = handler.call(@parser.http_method, @url, @headers, @body)
       end
+
       if result
         Billy.log(:info, "STUB #{@parser.http_method} #{@url}")
         response = EM::DelegatedHttpResponse.new(self)
         response.status = result[0]
         response.headers = result[1].merge('Connection' => 'close')
         response.content = result[2]
         response.send_response
-      elsif @parser.http_method == 'GET' && cache.cached?(@url)
+      elsif cache.cached?(@parser.http_method.downcase, @url, @body)
         Billy.log(:info, "CACHE #{@parser.http_method} #{@url}")
         respond_from_cache
       else
@@ -114,9 +115,10 @@ def proxy_request
         res_headers = res_headers.merge('Connection' => 'close')
         res_headers.delete('Transfer-Encoding')
         res_content = req.response.force_encoding('BINARY')
-        if @parser.http_method == 'GET' && cache.cacheable?(@url, res_headers)
-          cache.store(@url, res_status, res_headers, res_content)
+        if cache.cacheable?(@url, res_headers)
+          cache.store(@parser.http_method.downcase, @url, @body, res_status, res_headers, res_content)
         end
+
         res = EM::DelegatedHttpResponse.new(self)
         res.status = res_status
         res.headers = res_headers
@@ -126,7 +128,7 @@ def proxy_request
     end
 
     def respond_from_cache
-      cached_res = cache.fetch(@url)
+      cached_res = cache.fetch(@parser.http_method.downcase, @url, @body)
       res = EM::DelegatedHttpResponse.new(self)
       res.status = cached_res[:status]
       res.headers = cached_res[:headers]

puffing-billy.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "selenium-webdriver"
   gem.add_development_dependency "capybara-webkit"
   gem.add_development_dependency "rack"
+  gem.add_development_dependency "guard"
+  gem.add_development_dependency "rb-inotify"
   gem.add_runtime_dependency "eventmachine"
   gem.add_runtime_dependency "em-http-request"
   gem.add_runtime_dependency "eventmachine_httpserver"

spec/requests/examples/facebook_api_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'base64'
 
-describe 'Facebook API example', :type => :request, :js => true do
+describe 'Facebook API example', :type => :feature, :js => true do
   before do
     proxy.stub('https://www.facebook.com:443/dialog/oauth').and_return(Proc.new { |params,_,_|
       # mock a signed request from facebook.  the JS api never verifies the
@@ -20,4 +20,3 @@
     page.should have_content "Hi, Tester 1"
   end
 end
-

spec/requests/examples/tumblr_api_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe 'Tumblr API example', :type => :request, :js => true do
+describe 'Tumblr API example', :type => :feature, :js => true do
   before do
     proxy.stub('http://blog.howmanyleft.co.uk/api/read/json').and_return(
       :jsonp => {
@@ -27,4 +27,3 @@
     page.should have_content('News item 2 content here')
   end
 end
-