Getting started with Crawler is easy. The main class you need to care about is crawler.main.Crawler
crawler.main.Crawler
from crawler.utils import should_ignore
>>> should_ignore(['blog/$'], 'http://ericholscher.com/blog/') True
>>> should_ignore(['home'], 'http://ericholscher.com/blog/') True
from crawler.utils import log
>>> log('http://ericholscher.com/blog/', 200) OK: 200 http://ericholscher.com/blog/
>>> log('http://ericholscher.com/blog/', 500) ERR: 500 http://ericholscher.com/blog/
>>> log('http://ericholscher.com/blog/', 500) OK: 500 http://ericholscher.com/blog/