-
Notifications
You must be signed in to change notification settings - Fork 0
/
PageParser.py
36 lines (31 loc) · 1.21 KB
/
PageParser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
import requests
import time
from bs4 import BeautifulSoup
from random import randint
class PageParser:
def __init__(self, url, max_retry=3, max_wait_timeout=5):
self.url = url
self.cookie_jar = None
self.request_body = {}
self.max_retry = max_retry
self.current_page = None
self.dom = None
self.max_wait_timeout = max_wait_timeout
def _request(self):
self.current_page = None
self.dom = None
i = 1
while not self.current_page and i < self.max_retry:
try:
session = requests.Session()
if not self.cookie_jar:
session.get(self.url)
self.cookie_jar = session.cookies.get_dict()
request = requests.Request('POST', self.url, data=self.request_body, cookies=self.cookie_jar)
self.current_page = session.send(request.prepare())
i += 1
except requests.exceptions.ConnectionError:
time.sleep(randint(1, self.max_wait_timeout))
if not self.current_page:
raise ConnectionError(self.current_page)
self.dom = BeautifulSoup(self.current_page.content, 'lxml')