Skip to content
Browse files

304 cached

  • Loading branch information...
1 parent d9a9455 commit 1b50dd061f97ffa26eaa0b82d92b439c2a0d5ae7 @marsz committed Feb 19, 2012
Showing with 18 additions and 2 deletions.
  1. +8 −2 app/controllers/crawler_controller.rb
  2. +10 −0 spec/controllers/crawler_controller_spec.rb
View
10 app/controllers/crawler_controller.rb
@@ -26,8 +26,14 @@ def handle_by_action
end
def handle_download spider
spider.referer = params[:referer]
- url = spider.download(params[:url])
- {:url => url, :status => spider.response_code}
+ if s = Storage.find_by_source_url(params[:url])
+ url = s.url
+ status = 304
+ else
+ url = spider.download(params[:url])
+ status = spider.response_code
+ end
+ {:url => url, :status => status}
end
def handle_fetch spider
data = spider.fetch(params[:url], params[:query], :encoding => @options[:encoding])
View
10 spec/controllers/crawler_controller_spec.rb
@@ -106,6 +106,16 @@
hash["url"].size.should > 0
hash["status"].should == 200
end
+ it "304 cached" do
+ get "download", :format => :json, :url => "http://f2.urcosme.com/images/logo.gif",:token => @app.token
+ size = Storage.scoped.count
+ log_id = DomainCrawling.last.id
+ get "download", :format => :json, :url => "http://f2.urcosme.com/images/logo.gif",:token => @app.token
+ Storage.scoped.count.should == size
+ DomainCrawling.last.id.should == log_id
+ hash = ActiveSupport::JSON.decode(response.body)
+ hash["status"].should == 304
+ end
it "https" do
get "download", :format => :json, :url => "https://secure.gravatar.com/avatar/0b2f434918eb4a08439d180a13829631",:token => @app.token
response.should be_success

0 comments on commit 1b50dd0

Please sign in to comment.
Something went wrong with that request. Please try again.