Skip to content
Permalink
Browse files

Merge branch '0.16' of github.com:scrapy/scrapy into 0.16

  • Loading branch information...
Pablo Hoffman
Pablo Hoffman committed Nov 10, 2012
2 parents 7184094 + 1787011 commit d5087b0afa343426c423c20538667b9d3277b521
Showing with 112 additions and 61 deletions.
  1. +11 −0 docs/news.rst
  2. +2 −2 scrapy/__init__.py
  3. +9 −1 scrapy/commands/check.py
  4. +57 −38 scrapy/contracts/__init__.py
  5. +1 −4 scrapy/exceptions.py
  6. +32 −16 scrapy/tests/test_contracts.py
@@ -3,6 +3,17 @@
Release notes
=============

0.16.2 (released 2012-11-09)
----------------------------

- scrapy contracts: python2.6 compat (:commit:`a4a9199`)
- scrapy contracts verbose option (:commit:`ec41673`)
- proper unittest-like output for scrapy contracts (:commit:`86635e4`)
- added open_in_browser to debugging doc (:commit:`c9b690d`)
- removed reference to global scrapy stats from settings doc (:commit:`dd55067`)
- Fix SpiderState bug in Windows platforms (:commit:`58998f4`)


0.16.1 (released 2012-10-26)
----------------------------

@@ -2,8 +2,8 @@
Scrapy - a screen scraping framework written in Python
"""

version_info = (0, 16, 1)
__version__ = "0.16.1"
version_info = (0, 16, 2)
__version__ = "0.16.2"

import sys, os, warnings

@@ -1,11 +1,14 @@
from collections import defaultdict
from functools import wraps
from unittest import TextTestRunner

from scrapy import signals
from scrapy.command import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.conf import build_component_list
from scrapy.xlib.pydispatch import dispatcher


def _generate(cb):
@@ -31,6 +34,8 @@ def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=1, action="count",
help="print all contract hooks")

def run(self, args, opts):
# load contracts
@@ -39,6 +44,7 @@ def run(self, args, opts):
self.settings['SPIDER_CONTRACTS'],
)
self.conman = ContractsManager([load_object(c) for c in contracts])
self.results = TextTestRunner(verbosity=opts.verbose)._makeResult()

# contract requests
contract_reqs = defaultdict(list)
@@ -61,6 +67,8 @@ def run(self, args, opts):
for method in sorted(methods):
print ' * %s' % method
else:
dispatcher.connect(self.results.printErrors,
signals.engine_stopped)
self.crawler.start()

def get_requests(self, spider):
@@ -69,7 +77,7 @@ def get_requests(self, spider):
for key, value in vars(type(spider)).items():
if callable(value) and value.__doc__:
bound_method = value.__get__(spider, type(spider))
request = self.conman.from_method(bound_method)
request = self.conman.from_method(bound_method, self.results)

if request:
request.callback = _generate(request.callback)
@@ -1,10 +1,11 @@
import sys
import re
from functools import wraps
from unittest import TestCase

from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
from scrapy.exceptions import ContractFail


class ContractsManager(object):
@@ -27,7 +28,7 @@ def extract_contracts(self, method):

return contracts

def from_method(self, method, fail=False):
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
# calculate request args
@@ -43,9 +44,9 @@ def from_method(self, method, fail=False):

# execute pre and post hooks in order
for contract in reversed(contracts):
request = contract.add_pre_hook(request, fail)
request = contract.add_pre_hook(request, results)
for contract in contracts:
request = contract.add_post_hook(request, fail)
request = contract.add_post_hook(request, results)

return request

@@ -54,49 +55,67 @@ class Contract(object):
""" Abstract class for contracts """

def __init__(self, method, *args):
self.method = method
self.testcase_pre = self.create_testcase(method, 'pre-hook')
self.testcase_post = self.create_testcase(method, 'post-hook')
self.args = args

def add_pre_hook(self, request, fail=False):
cb = request.callback

@wraps(cb)
def wrapper(response):
try:
self.pre_process(response)
except ContractFail as e:
if fail:
raise
def create_testcase(self, method, hook):
spider = method.__self__.name

class ContractTestCase(TestCase):
def __str__(_self):
return "[%s] %s (@%s %s)" % (spider, method.__name__, self.name, hook)

name = '%s_%s' % (spider, method.__name__)
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback

@wraps(cb)
def wrapper(response):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
print e.format(self.method)
return list(iterate_spider_output(cb(response)))
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response)))

request.callback = wrapper

request.callback = wrapper
return request

def add_post_hook(self, request, fail=False):
cb = request.callback

@wraps(cb)
def wrapper(response):
output = list(iterate_spider_output(cb(response)))
try:
self.post_process(output)
except ContractFail as e:
if fail:
raise
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback

@wraps(cb)
def wrapper(response):
try:
output = list(iterate_spider_output(cb(response)))
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
print e.format(self.method)
return output
results.addSuccess(self.testcase_post)
finally:
return output

request.callback = wrapper

request.callback = wrapper
return request

def adjust_request_args(self, args):
return args

def pre_process(self, response):
pass

def post_process(self, output):
pass
@@ -52,7 +52,4 @@ class ScrapyDeprecationWarning(Warning):

class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""

def format(self, method):
return '[FAILED] %s:%s\n>>> %s\n' % \
(method.im_class.name, method.__name__, self)
pass
@@ -1,9 +1,10 @@
from unittest import TextTestRunner

from twisted.trial import unittest

from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.exceptions import ContractFail
from scrapy.contracts import ContractsManager
from scrapy.contracts.default import (
UrlContract,
@@ -71,54 +72,69 @@ def parse_no_url(self, response):
class ContractsManagerTest(unittest.TestCase):
contracts = [UrlContract, ReturnsContract, ScrapesContract]

def setUp(self):
self.conman = ContractsManager(self.contracts)
self.results = TextTestRunner()._makeResult()
self.results.stream = None

def should_succeed(self):
self.assertFalse(self.results.failures)
self.assertFalse(self.results.errors)

def should_fail(self):
self.assertTrue(self.results.failures)
self.assertFalse(self.results.errors)

def test_contracts(self):
conman = ContractsManager(self.contracts)
spider = TestSpider()

# extract contracts correctly
contracts = conman.extract_contracts(TestSpider.returns_request)
contracts = self.conman.extract_contracts(spider.returns_request)
self.assertEqual(len(contracts), 2)
self.assertEqual(frozenset(map(type, contracts)),
frozenset([UrlContract, ReturnsContract]))

# returns request for valid method
request = conman.from_method(TestSpider.returns_request)
request = self.conman.from_method(spider.returns_request, self.results)
self.assertNotEqual(request, None)

# no request for missing url
request = conman.from_method(TestSpider.parse_no_url)
request = self.conman.from_method(spider.parse_no_url, self.results)
self.assertEqual(request, None)

def test_returns(self):
conman = ContractsManager(self.contracts)

spider = TestSpider()
response = ResponseMock()

# returns_item
request = conman.from_method(spider.returns_item, fail=True)
request = self.conman.from_method(spider.returns_item, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [TestItem])
self.should_succeed()

# returns_request
request = conman.from_method(spider.returns_request, fail=True)
request = self.conman.from_method(spider.returns_request, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [Request])
self.should_succeed()

# returns_fail
request = conman.from_method(spider.returns_fail, fail=True)
self.assertRaises(ContractFail, request.callback, response)
request = self.conman.from_method(spider.returns_fail, self.results)
request.callback(response)
self.should_fail()

def test_scrapes(self):
conman = ContractsManager(self.contracts)

spider = TestSpider()
response = ResponseMock()

# scrapes_item_ok
request = conman.from_method(spider.scrapes_item_ok, fail=True)
request = self.conman.from_method(spider.scrapes_item_ok, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [TestItem])
self.should_succeed()

# scrapes_item_fail
request = conman.from_method(spider.scrapes_item_fail, fail=True)
self.assertRaises(ContractFail, request.callback, response)
request = self.conman.from_method(spider.scrapes_item_fail,
self.results)
request.callback(response)
self.should_fail()

0 comments on commit d5087b0

Please sign in to comment.
You can’t perform that action at this time.