@@ -10,7 +10,8 @@ class BotFilter::LinkTitle < BotFilter::Template title = data[:name] unless title begin - open(data[:url]) do |f| + open(data[:url], { 'User-Agent' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', + 'Referer' => data[:url].sub(%r{^([a-z]+:/+[^/]+)/.*$}, '\1/') }) do |f| title = f.read.match(/<title>(.*?)<\/title>/mi)[1].strip.gsub(/\s+/, ' ') end rescue lib/filters/link_title.rb @@ -44,10 +44,21 @@ describe BotFilter::LinkTitle do end it 'should use open to fetch URL' do - @filter.expects(:open).with(@url) + @filter.expects(:open).with(@url, anything) @filter.process({:url => @url, :type => :link}) end + it 'should specify a common browser as the user agent when opening' do + @filter.expects(:open).with(@url, has_entry('User-Agent', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)')) + @filter.process({:url => @url, :type => :link}) + end + + it 'should use the protocol and domain of the link as the HTTP referer when opening' do + url = 'http://bp1.blogspot.com/foo/bar/baz/dontlinkthisimage.jpg' + @filter.expects(:open).with(url, has_entry('Referer', 'http://bp1.blogspot.com/')) + @filter.process({:url => url, :type => :link}) + end + it 'should do nothing for non-links' do @filter.expects(:open).never data = {:url => @url} spec/bot_filter_link_title_spec.rb f97a3d87ca4584799c07a7cf0f5cb0657f7b7182 Rick Bradley rick@rickbradley.com http://github.com/flogic/nihilist_bot/commit/c67a6184200e783d10d48148452408cf73c2ff0c c67a6184200e783d10d48148452408cf73c2ff0c 2009-05-23T08:53:55-07:00 2009-05-23T08:53:55-07:00 Forge REFERER and User-Agent when fetching links a2e883846f53fe43952e5041c960f83292a83225 Rick Bradley rick@rickbradley.com