-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
executable file
·31 lines (27 loc) · 864 Bytes
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# -*- coding: utf-8 -*-
from scrapy.cmdline import execute
import sys
import os
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
execute(["scrapy", "crawl", "jobbole"])
# execute(["scrapy", "crawl", "zhihu"])
# execute(["scrapy", "crawl", "lagou"])
# import scrapy
# from scrapy.crawler import CrawlerProcess
# from spiders.jobbole import JobboleSpider
# from spiders.lagou import LagouSpider
# from twisted.internet import reactor, defer
# from scrapy.crawler import CrawlerRunner
# from scrapy.utils.log import configure_logging
#
# configure_logging()
# runner = CrawlerRunner()
#
# @defer.inlineCallbacks
# def crawl():
# yield runner.crawl(JobboleSpider)
# yield runner.crawl(LagouSpider)
# reactor.stop()
#
# crawl()
# reactor.run() # the script will block here until the last crawl call is finished