Merge 15543ed into 9896b76

kingfisher_scrapy/base_spider.py

@@ -8,7 +8,7 @@
 from jsonpointer import resolve_pointer
 
 from kingfisher_scrapy import util
-from kingfisher_scrapy.exceptions import KingfisherScrapyError, SpiderArgumentError
+from kingfisher_scrapy.exceptions import LinksSpiderError, SpiderArgumentError
 from kingfisher_scrapy.items import File, FileError, FileItem
 from kingfisher_scrapy.util import handle_http_error
 
@@ -379,4 +379,4 @@ def next_link(self, response):
             return self.build_request(url, formatter=self.next_page_formatter)
 
         if response.meta['depth'] == 0:
-            raise KingfisherScrapyError('next link not found on the first page: {}'.format(response.url))
+            raise LinksSpiderError('next link not found on the first page: {}'.format(response.url))

kingfisher_scrapy/exceptions.py

@@ -12,3 +12,7 @@ class SpiderArgumentError(KingfisherScrapyError):
 
 class MissingRequiredFieldError(KingfisherScrapyError, KeyError):
     """Raised when an item is missing a required field"""
+
+
+class LinksSpiderError(KingfisherScrapyError):
+    """Raised when a next link is invalid in LinksSpider"""

tests/__init__.py

@@ -7,8 +7,10 @@
 from kingfisher_scrapy.base_spider import BaseSpider
 
 
-def response_fixture(**kwargs):
-    request = Request('http://example.com', meta={'file_name': 'test', 'depth': 0})
+def response_fixture(meta=None, **kwargs):
+    if meta is None:
+        meta = {'file_name': 'test'}
+    request = Request('http://example.com', meta=meta)
     if 'status' not in kwargs:
         kwargs['status'] = 200
     if 'body' not in kwargs:

tests/test_links_spider.py

@@ -65,7 +65,13 @@ def test_parse_200():
 def test_next_link_not_found():
     spider = spider_with_crawler(spider_class=LinksSpider)
     spider.next_page_formatter = lambda url: 'next.json'
+    body = '{"links": {"next": ""}}'
 
     with pytest.raises(KingfisherScrapyError) as e:
-        assert spider.next_link(response_fixture(body='{"links": {"next": ""}}'))
+        meta = {'file_name': 'test', 'depth': 0}
+        assert spider.next_link(response_fixture(meta=meta, body=body))
     assert str(e.value) == 'next link not found on the first page: http://example.com'
+
+    meta = {'file_name': 'test', 'depth': 10}
+    response = spider.next_link(response_fixture(meta=meta, body=body))
+    assert response is None