-
Notifications
You must be signed in to change notification settings - Fork 0
/
Archive.py
223 lines (206 loc) · 9.4 KB
/
Archive.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
#!/usr/bin/env python
import sys
import re
import os
import json
import time
import threading
import dateutil.parser
import cpapi
import cputils
import logbook
LOGGER = logbook.Logger('archive')
LOG = logbook.FileHandler('monitoring.log')
LOG.push_application()
START_TIME = time.time()
class CmdLine:
def __init__(self):
self.auth_filename = "archive.auth"
self.report_module = "sva,csm"
self.url_base = "https://api.cloudpassage.com"
self.output_path = os.getcwd()
self.allowed_report_type = ["sva", "csm"]
def process_args(self, argv):
all_ok = True
self.progdir = os.path.dirname(sys.argv[0])
for arg in argv[1:]:
if arg.startswith("--auth="):
self.auth_filename = arg.split("=")[1]
elif arg.startswith("--base="):
self.base = arg.split("=")[1]
elif arg.startswith("--reportType"):
self.report_module = arg.split("=")[1]
cputils.verify_module(self.report_module, self.allowed_report_type)
elif arg.startswith("--output_path="):
self.output_path = os.path.abspath(arg.split("=")[1])
elif (arg == "-h") or (arg == "-?"):
all_ok = False
else:
print >>sys.stderr, "Unknown argument: %s" % arg
all_ok = False
return all_ok
def usage(self, progname):
print >> sys.stderr, "Usage: %s [flag] [...]" % os.path.basename(progname)
print >> sys.stderr, "Where flag is one or more of the following options:"
print >> sys.stderr, "--auth=<filename>\tSpecify name of file containing API credentials"
print >> sys.stderr, "--base=<url>\t\tSpecify the URL of the Halo REST API"
print >> sys.stderr, "--output_path=<file_path>\t\t\
Specify the file directory for archived.\
The default file path is the same as Archive_issues"
class ArchiveData:
def __init__(self):
self.api = cpapi.CPAPI()
print "Saving output files to %s" % CMD.output_path
def list_servers(self):
count = 1
server_list = []
finish = False
url = "%s:%d/v1/servers?per_page=5&page=1" %(self.api.base_url, self.api.port)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
if data:
LOGGER.info("First API was successful! Data is good.")
while (data is None) and (count < 4):
LOGGER.warn("Failed to connect to %s" % url)
self.api.authenticateClient()
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
LOGGER.warn(err_msg)
LOGGER.warn("retry: %d time" % count + "on %d" % url)
if data:
LOGGER.info("Successfully retreive server list from %d" % url)
count += 1
while(data != None) and (finish == False):
if 'servers' in data:
list_servers = json.loads(data)
servers = list_servers['servers']
for server in servers:
server_list.append((server['id'], server))
if 'pagination' in list_servers:
if 'next' in list_servers['pagination']:
url = list_servers['pagination']['next']
count_pagingation = 1
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
while (data is None) and (count_pagingation < 4):
self.api.authenticateClient()
LOGGER.warn(err_msg)
LOGGER.warn("retry: %d time" % count_pagingation + "on %s" % url)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
if data:
LOGGER.info("Successfully retreive server list from %s" % url)
count_pagingation += 1
if count == 4:
LOGGER.warn("Failed to connect to", url)
else:
finish = True
else:
finish = True
return server_list
def get_server_module(self, server_list, module):
server_id = server_list[0]
server_info = server_list[1]
new_findings = []
count = 1
url = "%s:%d/v1/servers/%s/%s" % (self.api.base_url, self.api.port, server_id, module)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
while (data is None) and (count < 4):
self.api.authenticateClient()
LOGGER.warn(err_msg)
LOGGER.warn("retry: %d time" % count + "on %s" % url)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
if data:
LOGGER.info("Successfully retreive server issue from %s" % url)
count += 1
if data:
scan_data = json.loads(data)
if 'scan' in scan_data:
findings = scan_data['scan']['findings']
scan_time = dateutil.parser.parse(scan_data['scan']['created_at'])
if findings:
for finding in findings:
if finding['status'] == 'bad':
new_findings.append(finding)
if new_findings:
if module == 'svm':
for new_finding in new_findings:
new_finding['age'] = self.get_issue_duration("sva", new_finding['package_name'], scan_data['id'])
if module == "sca":
for new_finding in new_findings:
new_finding['age'] = self.get_issue_duration("csm", new_finding['rule_name'], scan_data['id'])
scan_data['scan']['findings'] = new_findings
cputils.write_file(CMD.output_path, scan_time, scan_data, False)
LOGGER.info("Successfully archive %s scan from: %s" % (module, url))
if server_info:
cputils.write_file(CMD.output_path, scan_time, server_info, True)
LOGGER.info("Successfully retreive server data from %s" % server_id)
else:
LOGGER.warn("Failed to connect to %s" % url)
return None
def get_issue_duration(self, module, name, agent_id):
count = 1
duration = None
url = "%s:%d/v2/issues?issue_type=%s&name=%s&agent_id=%s" % (self.api.base_url, self.api.port, module, name, agent_id)
url = re.sub(r"\s+", '%20', url)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
while (data is None) and (count < 4):
self.api.authenticateClient()
LOGGER.warn(err_msg)
LOGGER.warn("retry: %d time" % count + "on %s" % url)
(data, auth_error, err_msg) = self.api.doGetRequest(url, self.api.authToken)
if data:
LOGGER.info("Successfully retreive issue from %s" % url)
count += 1
if data:
issue_data = json.loads(data)
if 'issues' in issue_data:
issues = issue_data['issues']
for issue in issues:
created_at = dateutil.parser.parse(issue['created_at'])
last_seen = dateutil.parser.parse(issue['last_seen_at'])
duration = last_seen - created_at
return str(duration.days)
def multi_threading(self, function, arg1, arg2):
threads = []
for i in arg1:
thread = threading.Thread(target=function, args=(i, arg2))
threads.append(thread)
thread.start()
thread.join()
return thread
def chunk(self, length, number):
x = []
for i in xrange(0, len(length), number):
x.append(length[i:i+number])
return x
def run(self, cmd):
threads = []
(credential_list, err_msg) = cputils.processAuthFile(cmd.auth_filename, cmd.progdir)
if err_msg:
print >> sys.stderr, err_msg
return False
if len(credential_list) < 1:
return False
# print credentials
credentials = credential_list[0]
self.api.base_url = cmd.url_base
self.api.key_id = credentials['id']
self.api.secret = credentials['secret']
resp = self.api.authenticateClient()
if not resp:
return False
print "Start archiving issues."
LOGGER.info("start archiving issues.")
server_list = self.list_servers()
print "--- %s servers ---" % (len(server_list))
LOGGER.info("--- %s servers ---" % (len(server_list)))
server_chuncks = self.chunk(server_list, 10)
for server_chunck in server_chuncks:
temp = self.multi_threading(self.get_server_module, server_chunck, 'sca')
temp = self.multi_threading(self.get_server_module, server_chunck, 'svm')
if __name__ == "__main__":
CMD = CmdLine()
if not CMD.process_args(sys.argv):
CMD.usage(sys.argv[0])
else:
REP = ArchiveData()
REP.run(CMD)
print "--- %s seconds ---" % (time.time() - START_TIME)
LOGGER.info("--- %s seconds ---" % (time.time() - START_TIME))