-
Notifications
You must be signed in to change notification settings - Fork 12
/
updatedocs.py
44 lines (32 loc) · 1.53 KB
/
updatedocs.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
import os
from itertools import groupby
from textwrap import dedent
from scrapy.commands import ScrapyCommand
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
class UpdateDocs(ScrapyCommand):
def short_desc(self):
return 'Update docs/spiders.rst'
def run(self, args, opts):
basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
def _keyfunc(module):
module_name = module.__name__.rsplit('.', 1)[-1]
if module_name.startswith(('costa_rica', 'dominican_republic')):
return '_'.join(module_name.split('_', 2)[:2])
return module_name.split('_', 1)[0]
with open(os.path.join(basedir, 'docs', 'spiders.rst'), 'w') as f:
f.write(dedent("""\
Spiders
=======
All the spiders have these common arguments:
sample
Sets the number of items to download.
.. Do not edit this file. Instead, run: `scrapy updatedocs`
"""))
for key, group in groupby(walk_modules('kingfisher_scrapy.spiders'), _keyfunc):
if key in ('spiders', 'fail'):
continue
f.write('\n{}\n{}\n'.format(key.replace('_', ' ').title(), '-' * len(key)))
for module in group:
for cls in iter_spider_classes(module):
f.write('\n.. autoclass:: {}.{}\n :no-members:\n'.format(module.__name__, cls.__name__))