/
settings.py
31 lines (26 loc) · 934 Bytes
/
settings.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
from itemadapter import ItemAdapter
from zyte_common_items import ZyteItemAdapter
ItemAdapter.ADAPTER_CLASSES.appendleft(ZyteItemAdapter)
BOT_NAME = "zyte_spider_templates_project"
SPIDER_MODULES = [
"zyte_spider_templates.spiders",
"zyte_spider_templates_project.spiders",
]
NEWSPIDER_MODULE = "zyte_spider_templates_project.spiders"
CLOSESPIDER_TIMEOUT_NO_ITEM = 900
ADDONS = {
"scrapy_zyte_api.Addon": 500,
}
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
SPIDER_MIDDLEWARES = {
"scrapy_poet.RetryMiddleware": 275,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"zyte_spider_templates.middlewares.AllowOffsiteMiddleware": 500,
"zyte_spider_templates.middlewares.CrawlingLogsMiddleware": 1000,
}
# scrapy-poet
SCRAPY_POET_DISCOVER = [
"zyte_spider_templates.pages",
"zyte_spider_templates_project.pages",
]