forked from scrapy/scrapyd
/
script.py
46 lines (40 loc) · 1.27 KB
/
script.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
"""This module can be used to execute Scrapyd from a Scrapy command"""
import sys
import os
from cStringIO import StringIO
from twisted.python import log
from twisted.internet import reactor
from twisted.application import app
from scrapy.utils.project import project_data_dir
from scrapy.exceptions import NotConfigured
from scrapyd import get_application
from scrapyd.config import Config
def _get_config():
datadir = os.path.join(project_data_dir(), 'scrapyd')
conf = {
'eggs_dir': os.path.join(datadir, 'eggs'),
'logs_dir': os.path.join(datadir, 'logs'),
'items_dir': os.path.join(datadir, 'items'),
'dbs_dir': os.path.join(datadir, 'dbs'),
}
for k in ['eggs_dir', 'logs_dir', 'items_dir', 'dbs_dir']: # create dirs
d = conf[k]
if not os.path.exists(d):
os.makedirs(d)
scrapyd_conf = """
[scrapyd]
eggs_dir = %(eggs_dir)s
logs_dir = %(logs_dir)s
items_dir = %(items_dir)s
dbs_dir = %(dbs_dir)s
""" % conf
return Config(extra_sources=[StringIO(scrapyd_conf)])
def execute():
try:
config = _get_config()
except NotConfigured:
config = None
log.startLogging(sys.stderr)
application = get_application(config)
app.startApplication(application, False)
reactor.run()