Skip to content

Commit

Permalink
Remove deprecated Spider.make_requests_from_url method
Browse files Browse the repository at this point in the history
  • Loading branch information
elacuesta committed Nov 20, 2019
1 parent 78ad016 commit 8d63cdd
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 49 deletions.
23 changes: 2 additions & 21 deletions scrapy/spiders/__init__.py
Expand Up @@ -4,14 +4,11 @@
See documentation in docs/topics/spiders.rst
"""
import logging
import warnings

from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import method_is_overridden


class Spider(object_ref):
Expand Down Expand Up @@ -57,24 +54,8 @@ def _set_crawler(self, crawler):
crawler.signals.connect(self.close, signals.spider_closed)

def start_requests(self):
cls = self.__class__
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
warnings.warn(
"Spider.make_requests_from_url method is deprecated; it "
"won't be called in future Scrapy releases. Please "
"override Spider.start_requests method instead (see %s.%s)." % (
cls.__module__, cls.__name__
),
)
for url in self.start_urls:
yield self.make_requests_from_url(url)
else:
for url in self.start_urls:
yield Request(url, dont_filter=True)

def make_requests_from_url(self, url):
""" This method is deprecated. """
return Request(url, dont_filter=True)
for url in self.start_urls:
yield Request(url, dont_filter=True)

def parse(self, response):
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
Expand Down
28 changes: 0 additions & 28 deletions tests/test_spider.py
Expand Up @@ -15,7 +15,6 @@
CSVFeedSpider, SitemapSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
from scrapy.utils.test import get_crawler


Expand Down Expand Up @@ -576,33 +575,6 @@ def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name='foo'), Spider)

def test_make_requests_from_url_deprecated(self):
class MySpider4(Spider):
name = 'spider1'
start_urls = ['http://example.com']

class MySpider5(Spider):
name = 'spider2'
start_urls = ['http://example.com']

def make_requests_from_url(self, url):
return Request(url + "/foo", dont_filter=True)

with warnings.catch_warnings(record=True) as w:
# spider without overridden make_requests_from_url method
# doesn't issue a warning
spider1 = MySpider4()
self.assertEqual(len(list(spider1.start_requests())), 1)
self.assertEqual(len(w), 0)

# spider with overridden make_requests_from_url issues a warning,
# but the method still works
spider2 = MySpider5()
requests = list(spider2.start_requests())
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0].url, 'http://example.com/foo')
self.assertEqual(len(w), 1)


class NoParseMethodSpiderTest(unittest.TestCase):

Expand Down

0 comments on commit 8d63cdd

Please sign in to comment.