[MRG+1] Redirect codes in meta

docs/topics/downloader-middleware.rst

@@ -725,9 +725,11 @@ RedirectMiddleware
    This middleware handles redirection of requests based on response status.
 
 .. reqmeta:: redirect_urls
+.. reqmeta:: redirect_reasons
 
-The urls which the request goes through (while being redirected) can be found
-in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
+The urls which the request goes through (while being redirected) and their 
+corresponding status can be found in the ``redirect_urls`` and ``redirect_reasons``
+:attr:`Request.meta <scrapy.http.Request.meta>` keys respectively.
 
 The :class:`RedirectMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
@@ -792,8 +794,9 @@ settings (see the settings documentation for more info):
 * :setting:`METAREFRESH_ENABLED`
 * :setting:`METAREFRESH_MAXDELAY`
 
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
 
 
 MetaRefreshMiddleware settings

docs/topics/request-response.rst

@@ -299,6 +299,7 @@ Those are:
 * :reqmeta:`dont_merge_cookies`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`

scrapy/downloadermiddlewares/redirect.py

@@ -34,6 +34,8 @@ def _redirect(self, redirected, request, spider, reason):
             redirected.meta['redirect_ttl'] = ttl - 1
             redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
                 [request.url]
+            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
+                [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
             logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",

tests/test_downloadermiddleware_redirect.py

@@ -139,6 +139,16 @@ def test_redirect_urls(self):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+    def test_redirect_reasons(self):
+        req1 = Request('http://scrapytest.org/first')
+        rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301)
+        req2 = self.mw.process_response(req1, rsp1, self.spider)
+        rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301)
+        req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+        self.assertEqual(req2.meta['redirect_reasons'], [301])
+        self.assertEqual(req3.meta['redirect_reasons'], [301, 301])
+
     def test_spider_handling(self):
         smartspider = self.crawler._create_spider('smarty')
         smartspider.handle_httpstatus_list = [404, 301, 302]
@@ -259,6 +269,15 @@ def test_redirect_urls(self):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+    def test_redirect_reasons(self):
+        req1 = Request('http://scrapytest.org/first')
+        rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected'))
+        req2 = self.mw.process_response(req1, rsp1, self.spider)
+        rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1'))
+        req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+        self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
+        self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
 
 if __name__ == "__main__":
     unittest.main()