forked from deibit/cansina
-
Notifications
You must be signed in to change notification settings - Fork 0
/
visitor.py
executable file
·156 lines (129 loc) · 5.04 KB
/
visitor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
import threading
import time
import sys
import urllib
try:
import requests
except ImportError:
print("Python module requests not found")
sys.exit(1)
SLEEP_TIME = 3
class Visitor(threading.Thread):
auth = None
user_agent = None
proxy = None
discriminator = None
banned_location = None
delay = 0
requests = ""
size_discriminator = -1
@staticmethod
def set_discriminator(discriminator):
Visitor.discriminator = discriminator
@staticmethod
def set_size_discriminator(size_discriminator):
Visitor.size_discriminator = int(size_discriminator)
@staticmethod
def set_banned_location(banned_location):
Visitor.banned_location = banned_location
@staticmethod
def set_user_agent(useragent):
Visitor.user_agent = useragent
@staticmethod
def set_proxy(proxy):
Visitor.proxy = proxy
@staticmethod
def set_delay(delay):
Visitor.delay = int(delay)
@staticmethod
def set_requests(type_request):
Visitor.requests = type_request
@staticmethod
def set_authentication(auth):
if auth:
Visitor.auth = tuple(auth.split(':'))
else:
Visitor.auth = auth
def __init__(self, number, payload, results):
threading.Thread.__init__(self)
self.number = number
self.payload = payload
self.results = results
self.__time = []
def run(self):
try:
while not self.payload.queue.empty():
self.visit(self.payload.queue.get())
self.payload.queue.task_done()
except AttributeError:
pass
def visit(self, task):
try:
headers = {}
if Visitor.user_agent:
headers = {"user-agent": Visitor.user_agent}
now = time.time()
if self.__time:
timeout = sum(self.__time) / len(self.__time)
else:
timeout = 10
r = None
if Visitor.proxy:
if Visitor.requests == "GET":
r = requests.get(task.get_complete_target(), headers=headers, proxies=Visitor.proxy, verify=False,
timeout=timeout, auth=Visitor.auth)
elif Visitor.requests == "HEAD":
r = requests.head(task.get_complete_target(), headers=headers, proxies=Visitor.proxy, verify=False,
timeout=timeout, auth=Visitor.auth)
else:
if Visitor.requests == "GET":
r = requests.get(task.get_complete_target(), headers=headers, verify=False, timeout=timeout,
auth=Visitor.auth)
elif Visitor.requests == "HEAD":
r = requests.head(task.get_complete_target(), headers=headers, verify=False, timeout=timeout,
auth=Visitor.auth)
after = time.time()
delta = (after - now) * 1000
tmp_content = r.content
task.response_size = len(tmp_content)
task.response_time = delta
self.__time.append(delta)
# If discriminator is found we mark it 404
if Visitor.discriminator and Visitor.discriminator in tmp_content:
r.status_code = '404'
# Check if the size of the page is set for discrimante fake 404 errors
if not Visitor.size_discriminator == -1 and task.response_size == Visitor.size_discriminator:
r.status_code = '404'
task.set_response_code(r.status_code)
# Look for interesting content
if task.content and (task.content in tmp_content) and not task.response_code == '404':
task.content_has_detected(True)
# Look for a redirection
if r.history and r.history[0]:
if r.url == task.get_complete_target() + '/':
pass
else:
# We dont want those pesky 404 relocations
task.set_location(r.url)
if task.location == self.banned_location:
task.set_response_code('404')
else:
task.set_response_code(r.history[0].status_code)
self.results.put(task)
if self.delay:
time.sleep(float(Visitor.delay / 0.001))
except requests.ConnectionError, requests.Timeout:
sys.stderr.write("Connection (or/and) timeout error")
except ValueError:
# Falling back to urllib (requests doesnt want freak chars)
now = time.time()
r = urllib.urlopen(task.get_complete_target(), proxies=self.proxy)
after = time.time()
delta = (after - now) * 1000
task.set_response_code(r.code)
c = r.readlines()
task.response_time = delta
task.response_size = len(c)
self.results.put(task)
except Exception as e:
print e.args