-
Notifications
You must be signed in to change notification settings - Fork 455
/
test_air_spider.py
50 lines (38 loc) · 1.19 KB
/
test_air_spider.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# -*- coding: utf-8 -*-
"""
Created on 2020/4/22 10:41 PM
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
import feapder
class TestAirSpider(feapder.AirSpider):
__custom_setting__ = dict(
USE_SESSION=True,
TASK_MAX_CACHED_SIZE=10,
)
def start_callback(self):
print("爬虫开始")
def end_callback(self):
print("爬虫结束")
def start_requests(self, *args, **kws):
for i in range(200):
print(i)
yield feapder.Request("https://www.baidu.com")
def download_midware(self, request):
# request.headers = {'User-Agent': ""}
# request.proxies = {"https":"https://12.12.12.12:6666"}
# request.cookies = {}
return request
def validate(self, request, response):
if response.status_code != 200:
raise Exception("response code not 200") # 重试
# if "哈哈" not in response.text:
# return False # 抛弃当前请求
def parse(self, request, response):
print(response.bs4().title)
print(response.xpath("//title").extract_first())
if __name__ == "__main__":
TestAirSpider(thread_count=1).start()