-
Notifications
You must be signed in to change notification settings - Fork 2.2k
/
sfp_grayhatwarfare.py
182 lines (146 loc) · 5.75 KB
/
sfp_grayhatwarfare.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
# -------------------------------------------------------------------------------
# Name: sfp_grayhatwarfare
# Purpose: Find bucket names matching the keyword extracted from a domain from Grayhat API.
#
# Author: <krishnasis@hotmail.com>
#
# Created: 24-01-2021
# Copyright: (c) Steve Micallef
# Licence: MIT
# -------------------------------------------------------------------------------
import json
import time
import urllib
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
class sfp_grayhatwarfare(SpiderFootPlugin):
meta = {
'name': "Grayhat Warfare",
'summary': "Find bucket names matching the keyword extracted from a domain from Grayhat API.",
'flags': ["apikey"],
'useCases': ["Footprint", "Investigate", "Passive"],
'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://buckets.grayhatwarfare.com/",
'model': "FREE_AUTH_LIMITED",
'references': [
"https://buckets.grayhatwarfare.com/docs/api/v1"
],
'apiKeyInstructions': [
"Visit https://grayhatwarfare.com/register",
"Register an account",
"Visit https://grayhatwarfare.com/account/settings",
"Your API key is listed under 'Api Key'",
],
'favIcon': "https://buckets.grayhatwarfare.com/assets/template/images/favicon.png",
'logo': "https://buckets.grayhatwarfare.com/assets/images/logo/logo-sm.png",
'description': "It is a searchable database of open buckets."
"Has up to million results of each bucket."
"Full text search with binary logic (can search for keywords and also stopwords)",
}
}
# Default options
opts = {
'api_key': '',
'per_page': 1000,
'max_pages': 2,
'pause': 1
}
# Option descriptions
optdescs = {
'api_key': 'Grayhat Warfare API key.',
'per_page': 'Maximum number of results per page (Max: 1000).',
'max_pages': 'Maximum number of pages to fetch.',
'pause': 'Number of seconds to wait between each API call.'
}
results = None
errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
# What events is this module interested in for input
def watchedEvents(self):
return [
"DOMAIN_NAME",
]
# What events this module produces
def producedEvents(self):
return [
'CLOUD_STORAGE_BUCKET',
'CLOUD_STORAGE_BUCKET_OPEN',
'RAW_RIR_DATA'
]
# Query Grayhat Warfare
def query(self, keyword, start):
params = urllib.parse.urlencode({
'keywords': keyword.encode('raw_unicode_escape'),
'access_token': self.opts['api_key']
})
headers = {
'Accept': 'application/json',
}
res = self.sf.fetchUrl(
f"https://buckets.grayhatwarfare.com/api/v1/buckets/{start}/{self.opts['per_page']}?{params}",
headers=headers,
timeout=15,
useragent=self.opts['_useragent'],
verify=True
)
time.sleep(self.opts['pause'])
if res['code'] != "200":
self.error("Unable to fetch data from Grayhat Warfare API.")
self.errorState = True
return None
if res['content'] is None:
self.debug('No response from Grayhat Warfare API.')
return None
try:
return json.loads(res['content'])
except Exception as e:
self.debug(f"Error processing JSON response: {e}")
return None
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
return
if self.errorState:
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.error("You enabled sfp_grayhatwarfare but did not set an API key!")
self.errorState = True
return
currentIndex = 0
currentPage = 0
maxPages = self.opts['max_pages']
perPage = self.opts['per_page']
keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
while currentPage < maxPages:
currentIndex = currentPage * perPage
if self.checkForStop():
return
if self.errorState:
break
data = self.query(keyword=keyword, start=currentIndex)
if not data:
return
for row in data.get('buckets'):
bucketName = row.get('bucket')
bucketKeyword = bucketName.split('.')[0]
self.debug(bucketKeyword)
if bucketKeyword.startswith(keyword) or bucketKeyword.endswith(keyword):
evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName, self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET_OPEN', f"{bucketName}: {row.get('fileCount')} files found.", self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event)
self.notifyListeners(evt)
currentPage += 1
if data.get('buckets_count') < perPage:
break
# End of sfp_grayhatwarfare class