-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
47 lines (36 loc) · 1.21 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
import logging
from waimaoba_crawler.pipeline.steps.get_urls import GetUrls
from waimaoba_crawler.pipeline.steps.get_company_info import GetCompanyInfo
from waimaoba_crawler.pipeline.steps.write_to_db import WriteToDB
from waimaoba_crawler.pipeline.steps.step import StepException
from waimaoba_crawler.pipeline.pipeline import Pipeline
def config_log():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s:%(asctime)s:%(message)s')
file_handler = logging.FileHandler('waimaoba.log')
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
logger.info('Log 設定已載入')
def main():
start_page = input('Start From: ')
end_page = input('End at: ')
inputs = {
'start_page': start_page,
'end_page': end_page,
}
steps = [
GetUrls(),
GetCompanyInfo(),
WriteToDB(),
]
p = Pipeline(steps)
p.run(inputs)
if __name__ == '__main__':
config_log()
main()