Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

44 lines (35 sloc) 1.306 kb
import os
from twisted.python.failure import Failure
SCRAPEDFMT = u"Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDFMT = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
CRAWLEDFMT = u"Crawled (%(status)s) %(request)s (referer: %(referer)s)%(flags)s"
class LogFormatter(object):
"""Class for generating log messages for different actions. All methods
must return a plain string which doesn't include the log level or the
timestamp
"""
def crawled(self, request, response, spider):
flags = ' %s' % str(response.flags) if response.flags else ''
return {
'format': CRAWLEDFMT,
'status': response.status,
'request': request,
'referer': request.headers.get('Referer'),
'flags': flags,
}
def scraped(self, item, response, spider):
src = response.getErrorMessage() if isinstance(response, Failure) else response
return {
'format': SCRAPEDFMT,
'src': src,
'item': item,
}
def dropped(self, item, exception, response, spider):
return {
'format': DROPPEDFMT,
'exception': exception,
'item': item,
}
@classmethod
def from_crawler(cls, crawler):
return cls()
Jump to Line
Something went wrong with that request. Please try again.