Skip to content
Permalink
Browse files

Test the parse command locally instead of against an external url

  • Loading branch information
curita authored and nramirezuy committed Jan 20, 2015
1 parent d0bf957 commit c3a6628138343ab00bb1fa8ba065ec350667212d
Showing with 21 additions and 9 deletions.
  1. +19 −9 scrapy/tests/test_commands.py
  2. +2 −0 scrapy/utils/testsite.py
@@ -7,9 +7,13 @@
from tempfile import mkdtemp

from twisted.trial import unittest
from twisted.internet import defer

from scrapy.utils.python import retry_on_eintr
from scrapy.utils.test import get_testenv
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest


class ProjectTest(unittest.TestCase):
project_name = 'testproject'
@@ -175,7 +179,9 @@ def test_runspider_unable_to_load(self):
self.assert_("Unable to load" in log)


class ParseCommandTest(CommandTest):
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):

command = 'parse'

def setUp(self):
super(ParseCommandTest, self).setUp()
@@ -215,17 +221,21 @@ def process_item(self, item, spider):
ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
""" % self.project_name)

@defer.inlineCallbacks
def test_spider_arguments(self):
p = self.proc('parse', '--spider', self.spider_name, '-a', 'test_arg=1',
'-c', 'parse', 'http://scrapinghub.com')
log = p.stderr.read()
self.assert_("[parse_spider] DEBUG: It Works!" in log, log)
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-a', 'test_arg=1',
'-c', 'parse',
self.url('/html')])
self.assert_("[parse_spider] DEBUG: It Works!" in stderr, stderr)

@defer.inlineCallbacks
def test_pipelines(self):
p = self.proc('parse', '--spider', self.spider_name, '--pipelines',
'-c', 'parse', 'http://scrapinghub.com')
log = p.stderr.read()
self.assert_("[scrapy] INFO: It Works!" in log, log)
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'--pipelines',
'-c', 'parse',
self.url('/html')])
self.assert_("[scrapy] INFO: It Works!" in stderr, stderr)


class BenchCommandTest(CommandTest):
@@ -7,10 +7,12 @@
class SiteTest(object):

def setUp(self):
super(SiteTest, self).setUp()
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
self.baseurl = "http://localhost:%d/" % self.site.getHost().port

def tearDown(self):
super(SiteTest, self).tearDown()
self.site.stopListening()

def url(self, path):

0 comments on commit c3a6628

Please sign in to comment.
You can’t perform that action at this time.