Skip to content

Commit

Permalink
Remove deprecated Spider.make_requests_from_url method
Browse files Browse the repository at this point in the history
  • Loading branch information
elacuesta committed Feb 3, 2020
1 parent 22f7934 commit c59560a
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 47 deletions.
23 changes: 3 additions & 20 deletions scrapy/spiders/__init__.py
Expand Up @@ -4,13 +4,11 @@
See documentation in docs/topics/spiders.rst
"""
import logging
import warnings

from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
from scrapy.utils.deprecate import method_is_overridden


class Spider(object_ref):
Expand Down Expand Up @@ -56,29 +54,14 @@ def _set_crawler(self, crawler):
crawler.signals.connect(self.close, signals.spider_closed)

def start_requests(self):
cls = self.__class__
if not self.start_urls and hasattr(self, 'start_url'):
raise AttributeError(
"Crawling could not start: 'start_urls' not found "
"or empty (but found 'start_url' attribute instead, "
"did you miss an 's'?)")
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
warnings.warn(
"Spider.make_requests_from_url method is deprecated; it "
"won't be called in future Scrapy releases. Please "
"override Spider.start_requests method instead (see %s.%s)." % (
cls.__module__, cls.__name__
),
)
for url in self.start_urls:
yield self.make_requests_from_url(url)
else:
for url in self.start_urls:
yield Request(url, dont_filter=True)

def make_requests_from_url(self, url):
""" This method is deprecated. """
return Request(url, dont_filter=True)

for url in self.start_urls:
yield Request(url, dont_filter=True)

def parse(self, response):
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
Expand Down
27 changes: 0 additions & 27 deletions tests/test_spider.py
Expand Up @@ -583,33 +583,6 @@ def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name='foo'), Spider)

def test_make_requests_from_url_deprecated(self):
class MySpider4(Spider):
name = 'spider1'
start_urls = ['http://example.com']

class MySpider5(Spider):
name = 'spider2'
start_urls = ['http://example.com']

def make_requests_from_url(self, url):
return Request(url + "/foo", dont_filter=True)

with warnings.catch_warnings(record=True) as w:
# spider without overridden make_requests_from_url method
# doesn't issue a warning
spider1 = MySpider4()
self.assertEqual(len(list(spider1.start_requests())), 1)
self.assertEqual(len(w), 0)

# spider with overridden make_requests_from_url issues a warning,
# but the method still works
spider2 = MySpider5()
requests = list(spider2.start_requests())
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0].url, 'http://example.com/foo')
self.assertEqual(len(w), 1)


class NoParseMethodSpiderTest(unittest.TestCase):

Expand Down

0 comments on commit c59560a

Please sign in to comment.