-
Notifications
You must be signed in to change notification settings - Fork 74
/
brightdata.py
30 lines (24 loc) · 809 Bytes
/
brightdata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
from urllib import request as urlrequest
import random
username = 'your_username'
password = 'your_password'
port = 22225
session_id = random.random()
super_proxy_url = ('http://%s-country-us-session-%s:%s@brd.superproxy.io:%d' %
(username, session_id, password, port))
proxy_handler = urlrequest.ProxyHandler({
'http': super_proxy_url,
'https': super_proxy_url,
})
opener = urlrequest.build_opener(proxy_handler)
print('Performing request')
URL = 'https://mathworld.wolfram.com/topics/FoundationsofMathematics.html'
response = opener.open(URL)
status_code = response.getcode()
page = response.read()
print(page)
print("status code: ", status_code)
# check which IP was used for crawling
URL = "https://ipecho.net/plain"
response = opener.open(URL)
print(response.read())