Skip to content
Permalink
Browse files

fix a reference to unexistent engine.slots. closes #593

  • Loading branch information
dangra committed Feb 14, 2014
1 parent 8ae11bf commit 13c099a104f433903d5f9ad9469a6de2fcd56062
Showing with 45 additions and 39 deletions.
  1. +12 −15 docs/topics/telnetconsole.rst
  2. +6 −0 scrapy/tests/spiders.py
  3. +15 −0 scrapy/tests/test_crawl.py
  4. +12 −24 scrapy/utils/engine.py
@@ -83,24 +83,21 @@ using the telnet console::
>>> est()
Execution engine status

time()-engine.start_time : 9.24237799644
time()-engine.start_time : 8.62972998619
engine.has_capacity() : False
engine.downloader.is_idle() : False
len(engine.downloader.slots) : 2
len(engine.downloader.active) : 16
engine.scraper.is_idle() : False

Spider: <GayotSpider 'gayotcom' at 0x2dc2b10>
engine.spider_is_idle(spider) : False
engine.slots[spider].closing : False
len(engine.slots[spider].inprogress) : 21
len(engine.slots[spider].scheduler.dqs or []) : 0
len(engine.slots[spider].scheduler.mqs) : 4453
len(engine.scraper.slot.queue) : 0
len(engine.scraper.slot.active) : 5
engine.scraper.slot.active_size : 1515069
engine.scraper.slot.itemproc_size : 0
engine.scraper.slot.needs_backout() : False
engine.spider.name : followall
engine.spider_is_idle(engine.spider) : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0
len(engine.slot.scheduler.mqs) : 92
len(engine.scraper.slot.queue) : 0
len(engine.scraper.slot.active) : 0
engine.scraper.slot.active_size : 0
engine.scraper.slot.itemproc_size : 0
engine.scraper.slot.needs_backout() : False


Pause, resume and stop the Scrapy engine
@@ -137,6 +137,8 @@ def parse(self, response):
class SingleRequestSpider(MetaSpider):

seed = None
callback_func = None
errback_func = None

def start_requests(self):
if isinstance(self.seed, Request):
@@ -146,8 +148,12 @@ def start_requests(self):

def parse(self, response):
self.meta.setdefault('responses', []).append(response)
if callable(self.callback_func):
return self.callback_func(response)
if 'next' in response.meta:
return response.meta['next']

def on_error(self, failure):
self.meta['failure'] = failure
if callable(self.errback_func):
return self.errback_func(failure)
@@ -192,3 +192,18 @@ def test_referer_header(self):
# last request explicitly sets a Referer header
echo3 = json.loads(spider.meta['responses'][3].body)
self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])

@defer.inlineCallbacks
def test_engine_status(self):
from scrapy.utils.engine import get_engine_status
est = []

def cb(response):
est.append(get_engine_status(spider.crawler.engine))

spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb)
yield docrawl(spider)
self.assertEqual(len(est), 1, est)
s = dict(est[0])
self.assertEqual(s['engine.spider.name'], spider.name)
self.assertEqual(s['len(engine.scraper.slot.active)'], 1)
@@ -5,14 +5,13 @@

def get_engine_status(engine):
"""Return a report of the current engine status"""
global_tests = [
tests = [
"time()-engine.start_time",
"engine.has_capacity()",
"len(engine.downloader.active)",
"engine.scraper.is_idle()",
]
spider_tests = [
"engine.spider_is_idle(spider)",
"engine.spider.name",
"engine.spider_is_idle(engine.spider)",
"engine.slot.closing",
"len(engine.slot.inprogress)",
"len(engine.slot.scheduler.dqs or [])",
@@ -24,34 +23,23 @@ def get_engine_status(engine):
"engine.scraper.slot.needs_backout()",
]

status = {'global': [], 'spiders': {}}
for test in global_tests:
checks = []
for test in tests:
try:
status['global'] += [(test, eval(test))]
checks += [(test, eval(test))]
except Exception as e:
status['global'] += [(test, "%s (exception)" % type(e).__name__)]
for spider in engine.slots.keys():
x = []
for test in spider_tests:
try:
x += [(test, eval(test))]
except Exception as e:
x += [(test, "%s (exception)" % type(e).__name__)]
status['spiders'][spider] = x
return status
checks += [(test, "%s (exception)" % type(e).__name__)]

return checks

def format_engine_status(engine=None):
status = get_engine_status(engine)
checks = get_engine_status(engine)
s = "Execution engine status\n\n"
for test, result in status['global']:
for test, result in checks:
s += "%-47s : %s\n" % (test, result)
s += "\n"
for spider, tests in status['spiders'].items():
s += "Spider: %s\n" % spider
for test, result in tests:
s += " %-50s : %s\n" % (test, result)

return s

def print_engine_status(engine):
print(format_engine_status(engine))

0 comments on commit 13c099a

Please sign in to comment.
You can’t perform that action at this time.