Merge pull request #9 from ephox/master

Improved handling of redirects
stewartmckee · Jul 17, 2012 · a92c337 · a92c337
2 parents ca6196a + 57da8c1
commit a92c337
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/lib/crawl_job.rb b/lib/crawl_job.rb
@@ -35,8 +35,11 @@ def self.perform(content_request)
         # if there is no limit or we're still under it lets get the url
         if within_crawl_limits?(content_request[:crawl_limit])
           begin
+            # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
             @redis.srem "queued", content_request[:url]
             @redis.sadd "crawled", content_request[:url]
+            @redis.srem "queued", content[:url]
+            @redis.sadd "crawled", content[:url]
             # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
             if content_request[:crawl_limit_by_page]
               if content[:mime_type].match("text/html")