From b0caccdc8954b128f5940d5ba60087929ff09321 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 12 Feb 2019 16:25:28 +0100 Subject: [PATCH 01/31] fixed issues plus jira comment formatting --- docker-compose.v6.yml | 4 ++-- docker-compose.yml | 2 +- vulnwhisp/reporting/jira_api.py | 18 +++++++++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docker-compose.v6.yml b/docker-compose.v6.yml index 205ce1d..58daac9 100644 --- a/docker-compose.v6.yml +++ b/docker-compose.v6.yml @@ -6,7 +6,7 @@ services: environment: - cluster.name=vulnwhisperer - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - xpack.security.enabled=false ulimits: @@ -21,7 +21,7 @@ services: - esdata1:/usr/share/elasticsearch/data ports: - 9200:9200 - restart: always + #restart: always networks: esnet: aliases: diff --git a/docker-compose.yml b/docker-compose.yml index 4f26266..61cdae3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: - 9200:9200 environment: - xpack.security.enabled=false - restart: always + #restart: always networks: esnet: aliases: diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 3c76b12..49fd0be 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -39,8 +39,8 @@ def create_ticket(self, title, desc, project="IS", components=[], tags=[]): for tag in tags: labels.append(str(tag)) - self.logger.info("creating ticket for project {} title[20] {}".format(project, title[:20])) - self.logger.info("project {} has a component requirement: {}".format(project, self.PROJECT_COMPONENT_TABLE[project])) + self.logger.info("creating ticket for project {} title: {}".format(project, title[:20])) + self.logger.info("project {} has a component requirement: {}".format(project, components)) project_obj = self.jira.project(project) components_ticket = [] for component in components: @@ -205,13 +205,21 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): difference = list(set(assets).symmetric_difference(ticket_assets)) comment = '' + added = '' + removed = '' #put a comment with the assets that have been added/removed for asset in difference: if asset in assets: - comment += "Asset {} have been added to the ticket as vulnerability *has been newly detected*.\n".format(asset) + if not added: + added = 'The following assets *have been newly detected*:\n' + added += '* {}\n'.format(asset) elif asset in ticket_assets: - comment += "Asset {} have been removed from the ticket as vulnerability *has been resolved*.\n".format(asset) - + if not removed: + removed= 'The following assets *have been resolved*:\n' + removed += '* {}\n'.format(asset) + + comment = added + removed + try: ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels}) self.logger.info("Ticket {} updated successfully".format(ticketid)) From ccf2e4b1d17facafd9140aa594140d3b0df028f0 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 12 Feb 2019 16:51:26 +0100 Subject: [PATCH 02/31] fix #147 --- vulnwhisp/vulnwhisp.py | 43 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index a728896..fd7c1aa 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -838,32 +838,29 @@ def whisper_reports(self, else: self.logger.info('Processing report ID: {}'.format(report_id)) vuln_ready = self.qualys_scan.process_data(scan_id=report_id) - if not vuln_ready.empty: - vuln_ready['scan_name'] = scan_name - vuln_ready['scan_reference'] = report_id - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + vuln_ready['scan_name'] = scan_name + vuln_ready['scan_reference'] = report_id + vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) - record_meta = ( - scan_name, - scan_reference, - launched_date, - report_name, - time.time(), - vuln_ready.shape[0], - self.CONFIG_SECTION, - report_id, - 1, - ) - self.record_insert(record_meta) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + vuln_ready.shape[0], + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) - if output_format == 'json': - with open(relative_path_name, 'w') as f: - f.write(vuln_ready.to_json(orient='records', lines=True)) - f.write('\n') + if output_format == 'json': + with open(relative_path_name, 'w') as f: + f.write(vuln_ready.to_json(orient='records', lines=True)) + f.write('\n') - self.logger.info('Report written to {}'.format(report_name)) - else: - return False + self.logger.info('Report written to {}'.format(report_name)) except Exception as e: self.logger.error('Could not process {}: {}'.format(report_id, str(e))) From 8c5398727025bcdb9da933e7d4a8d81822604a65 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 12 Feb 2019 16:56:00 +0100 Subject: [PATCH 03/31] tracking of processing was in debug instead of info logging --- vulnwhisp/vulnwhisp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index fd7c1aa..edd6e11 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -625,7 +625,7 @@ def process_web_assets(self): for app in self.scans_to_process.iterrows(): counter += 1 r = app[1] - self.logger.debug('Processing {}/{}'.format(counter, len(self.scans_to_process))) + self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process))) self.whisper_reports(report_id=r['id'], launched_date=r['launchedDate'], scan_name=r['name'], @@ -884,7 +884,7 @@ def process_vuln_scans(self): for app in self.scans_to_process.iterrows(): counter += 1 r = app[1] - self.logger.debug('Processing {}/{}'.format(counter, len(self.scans_to_process))) + self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process))) self.whisper_reports(report_id=r['id'], launched_date=r['date'], scan_name=r['name'], From bc3367e3109c7f2fe41adc9092c09c52f0df6d35 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 12 Feb 2019 18:01:46 +0100 Subject: [PATCH 04/31] exception of empty scans --- vulnwhisp/vulnwhisp.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index edd6e11..9d6da74 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -1041,8 +1041,12 @@ def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_cri risks = ['info', 'low', 'medium', 'high', 'critical'] # +1 as array is 0-4, but score is 1-5 min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1 - - data=[json.loads(line) for line in open(fullpath).readlines()] + + try: + data=[json.loads(line) for line in open(fullpath).readlines()] + except Exception as e: + self.logger.warn("Scan has no vulnerabilities, skipping.") + return vulnerabilities #qualys fields we want - [] for index in range(len(data)): @@ -1142,8 +1146,8 @@ def jira_sync(self, source, scan_name): self.jira.sync(vulnerabilities, project, components) else: - self.logger.info("Vulnerabilities from {source} has not been parsed! Exiting...".format(source=source)) - sys.exit(0) + self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name)) + return False return True From 177c2548baeabb0b5ff90856c94970ad2ab5eb2d Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 12 Feb 2019 18:18:24 +0100 Subject: [PATCH 05/31] allow jira sync module to run after the rest --- configs/frameworks_example.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 2f778ca..3d7d0ae 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -89,6 +89,7 @@ verbose=true #proxy_password = proxypass [jira] +enabled = false hostname = jira-host username = username password = password From 587546a72646ffef5af66f51a3932d3f68ac9a6d Mon Sep 17 00:00:00 2001 From: Quim Date: Thu, 14 Feb 2019 14:16:31 +0100 Subject: [PATCH 06/31] fix typo --- vulnwhisp/reporting/jira_api.py | 6 +++--- vulnwhisp/reporting/resources/ticket.tpl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 49fd0be..b6bf9f3 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -295,8 +295,8 @@ def is_risk_accepted(self, ticket_obj): if "risk_accepted" in labels: self.logger.warn("Ticket {} accepted risk, will be ignored".format(ticket_obj)) return True - elif "server_decomission" in labels: - self.logger.warn("Ticket {} server decomissioned, will be ignored".format(ticket_obj)) + elif "server_decommission" in labels: + self.logger.warn("Ticket {} server decommissioned, will be ignored".format(ticket_obj)) return True self.logger.info("Ticket {} risk has not been accepted".format(ticket_obj)) return False @@ -312,7 +312,7 @@ def reopen_ticket(self, ticketid): if self.is_ticket_reopenable(ticket_obj): comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported). In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. - If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it. + If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. If you have further doubts, please contact the Security Team.''' error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment) self.logger.info("Ticket {} reopened successfully".format(ticketid)) diff --git a/vulnwhisp/reporting/resources/ticket.tpl b/vulnwhisp/reporting/resources/ticket.tpl index f00d267..675a560 100644 --- a/vulnwhisp/reporting/resources/ticket.tpl +++ b/vulnwhisp/reporting/resources/ticket.tpl @@ -29,4 +29,4 @@ Please do not delete or modify the ticket assigned tags or title, as they are us In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. -If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it. +If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. From c2d80c7fced3ddf6dc96af39c30198b7f597e2c2 Mon Sep 17 00:00:00 2001 From: Quim Date: Fri, 15 Feb 2019 16:24:52 +0100 Subject: [PATCH 07/31] made host resolution optional from the config file with dns_resolv var --- configs/frameworks_example.ini | 1 + elk6/vulnwhisperer.ini | 1 + vulnwhisp/vulnwhisp.py | 52 ++++++++++++++++++++-------------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 3d7d0ae..77d283c 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -96,6 +96,7 @@ password = password write_path = /opt/VulnWhisperer/data/jira/ db_path = /opt/VulnWhisperer/data/database verbose = true +dns_resolv = False #Sample jira report scan, will automatically be created for existent scans #[jira.qualys_vuln.test_scan] diff --git a/elk6/vulnwhisperer.ini b/elk6/vulnwhisperer.ini index 12c2d7c..2b92761 100644 --- a/elk6/vulnwhisperer.ini +++ b/elk6/vulnwhisperer.ini @@ -95,6 +95,7 @@ password = password write_path = /opt/vulnwhisperer/data/jira/ db_path = /opt/vulnwhisperer/data/database verbose = true +dns_resolv = False #Sample jira report scan, will automatically be created for existent scans #[jira.qualys_vuln.test_scan] diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 9d6da74..287e919 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -983,8 +983,17 @@ def get_env_variables(self, source, scan_name): if not fullpath: self.logger.error('Scan file path "{scan_name}" for source "{source}" has not been found.'.format(scan_name=scan_name, source=source)) sys.exit(1) + + dns_resolv = self.config.get('jira','dns_resolv') + if dns_resolv in ('False', 'false', ''): + dns_resolv = False + elif dns_resolv in ('True', 'true'): + dns_resolv = True + else: + self.logger.error("dns_resolv variable not setup in [jira] section; will not do dns resolution") + dns_resolv = False - return project, components, fullpath, min_critical + return project, components, fullpath, min_critical, dns_resolv def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical): @@ -1033,7 +1042,7 @@ def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical return vulnerabilities - def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical): + def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False): #parsing of the qualys vulnerabilities schema #parse json vulnerabilities = [] @@ -1070,7 +1079,7 @@ def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_cri vuln['ips'] = [] #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n! - vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index]))) + vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) #different risk system than Nessus! vuln['risk'] = risks[int(data[index]['risk'])-1] @@ -1085,31 +1094,32 @@ def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_cri # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry for vuln in vulnerabilities: if vuln['title'] == data[index]['plugin_name']: - vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index]))) + vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) return vulnerabilities - def get_asset_fields(self, vuln): + def get_asset_fields(self, vuln, dns_resolv): values = {} values['ip'] = vuln['ip'] values['protocol'] = vuln['protocol'] values['port'] = vuln['port'] values['dns'] = '' - if vuln['dns']: - values['dns'] = vuln['dns'] - else: - if values['ip'] in self.host_resolv_cache.keys(): - self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip'])) - values['dns'] = self.host_resolv_cache[values['ip']] + if dns_resolv: + if vuln['dns']: + values['dns'] = vuln['dns'] else: - self.logger.debug("No hostname, trying to resolve {ip}'s hostname.".format(ip=values['ip'])) - try: - values['dns'] = socket.gethostbyaddr(vuln['ip'])[0] - self.host_resolv_cache[values['ip']] = values['dns'] - self.logger.debug("Hostname found: {hostname}.".format(hostname=values['dns'])) - except: - self.host_resolv_cache[values['ip']] = '' - self.logger.debug("Hostname not found for: {ip}.".format(ip=values['ip'])) + if values['ip'] in self.host_resolv_cache.keys(): + self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip'])) + values['dns'] = self.host_resolv_cache[values['ip']] + else: + self.logger.debug("No hostname, trying to resolve {ip}'s hostname.".format(ip=values['ip'])) + try: + values['dns'] = socket.gethostbyaddr(vuln['ip'])[0] + self.host_resolv_cache[values['ip']] = values['dns'] + self.logger.debug("Hostname found: {hostname}.".format(hostname=values['dns'])) + except: + self.host_resolv_cache[values['ip']] = '' + self.logger.debug("Hostname not found for: {ip}.".format(ip=values['ip'])) for key in values.keys(): if not values[key]: @@ -1127,7 +1137,7 @@ def parse_vulnerabilities(self, fullpath, source, scan_name, min_critical): def jira_sync(self, source, scan_name): self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name)) - project, components, fullpath, min_critical = self.get_env_variables(source, scan_name) + project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name) vulnerabilities = [] @@ -1137,7 +1147,7 @@ def jira_sync(self, source, scan_name): #***Qualys VM parsing*** if source == "qualys_vuln": - vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical) + vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv) #***JIRA sync*** if vulnerabilities: From 521184d079a58bfbccbb58dc80284cf908470508 Mon Sep 17 00:00:00 2001 From: Quim Montal Date: Thu, 21 Feb 2019 22:20:19 +0100 Subject: [PATCH 08/31] Update bug_report.md added debug trail request --- .github/ISSUE_TEMPLATE/bug_report.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a540ec1..b4c7dd4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,7 +11,10 @@ assignees: '' A clear and concise description of what the bug is. **Affected module** -Which one is the module that is not working as expected, e.g. Nessus, Qualys WAS, Qualys VM, OpenVAS, ELK, Jira...) +Which one is the module that is not working as expected, e.g. Nessus, Qualys WAS, Qualys VM, OpenVAS, ELK, Jira...). + +**VulnWhisperer debug trail** +If applicable, paste the VulnWhisperer debug trail of the execution for further detail (execute with '-d' flag). **To Reproduce** Steps to reproduce the behavior: @@ -27,8 +30,9 @@ A clear and concise description of what you expected to happen. If applicable, add screenshots to help explain your problem. **System in which VulnWhisperer runs (please complete the following information):** - - OS: [e.g. iOS] - - Version [e.g. 22] + - OS: [e.g. Ubuntu Server] + - Version: [e.g. 18.04.2 LTS] + - VulnWhisperer Version: [e.g. 1.7.1] **Additional context** Add any other context about the problem here. From 2c7965d2d9ff06e15b13990d72026b2cc8c1f895 Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 12:08:04 +0100 Subject: [PATCH 09/31] fix #151 --- vulnwhisp/reporting/jira_api.py | 120 +++++++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 17 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index b6bf9f3..04afe36 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -29,18 +29,20 @@ def __init__(self, hostname=None, username=None, password=None, path="", debug=F self.JIRA_RESOLUTION_FIXED = "Fixed" self.clean_obsolete = clean_obsolete self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl' + self.max_ips_ticket = 30 + self.attachment_filename = "vulnerable_assets.txt" if path: self.download_tickets(path) else: self.logger.warn("No local path specified, skipping Jira ticket download.") - def create_ticket(self, title, desc, project="IS", components=[], tags=[]): + def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents = []): labels = ['vulnerability_management'] for tag in tags: labels.append(str(tag)) self.logger.info("creating ticket for project {} title: {}".format(project, title[:20])) - self.logger.info("project {} has a component requirement: {}".format(project, components)) + self.logger.debug("project {} has a component requirement: {}".format(project, components)) project_obj = self.jira.project(project) components_ticket = [] for component in components: @@ -60,8 +62,12 @@ def create_ticket(self, title, desc, project="IS", components=[], tags=[]): issuetype={'name': 'Bug'}, labels=labels, components=components_ticket) - + self.logger.info("Ticket {} created successfully".format(new_issue)) + + if attachment_contents: + self.add_content_as_attachment(new_issue, attachment_contents) + return new_issue #Basic JIRA Metrics @@ -108,13 +114,18 @@ def sync(self, vulnerabilities, project, components=[]): self.ticket_update_assets(vuln, ticketid, ticket_assets) self.add_label(ticketid, vuln['risk']) continue - + attachment_contents = [] + # if assets >30, add as attachment + # create local text file with assets, attach it to ticket + if len(vuln['ips']) > self.max_ips_ticket: + attachment_contents = vuln['ips'] + vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))] try: tpl = template(self.template_path, vuln) except Exception as e: self.logger.error('Exception templating: {}'.format(str(e))) return 0 - self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']]) + self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents) self.close_fixed_tickets(vulnerabilities) # we reinitialize so the next sync redoes the query with their specific variables @@ -159,12 +170,72 @@ def ticket_get_unique_fields(self, ticket): try: affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0] assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section))) - except: - self.logger.error("Ticket IPs regex failed. Ticket ID: {}".format(ticketid)) + + if not assets: + #check if attachment, if so, get assets from attachment + affected_assets_section = self.check_ips_attachment(ticket) + if affected_assets_section: + assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section))) + + except Exception as e: + self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e)) assets = [] return ticketid, title, assets + def check_ips_attachment(self, ticket): + affected_assets_section = [] + try: + fields = self.jira.issue(ticket.key).raw.get('fields') + attachments = fields.get('attachment') + affected_assets_section = "" + #we will make sure we get the latest version of the file + latest = '' + attachment_id = '' + if attachments: + for item in attachments: + if item.get('filename') == self.attachment_filename: + if not latest: + latest = item.get('created') + attachment_id = item.get('id') + else: + if latest < item.get('created'): + latest = item.get('created') + attachment_id = item.get('id') + affected_assets_section = self.jira.attachment(attachment_id).get() + + except Exception as e: + self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e)) + + return affected_assets_section + + def clean_old_attachments(self, ticket): + fields = ticket.raw.get('fields') + attachments = fields.get('attachment') + if attachments: + for item in attachments: + if item.get('filename') == self.attachment_filename: + self.jira.delete_attachment(item.get('id')) + + def add_content_as_attachment(self, issue, contents): + try: + #Create the file locally with the data + attachment_file = open(self.attachment_filename, "w") + attachment_file.write("\n".join(contents)) + attachment_file.close() + #Push the created file to the ticket + attachment_file = open(self.attachment_filename, "rb") + self.jira.add_attachment(issue, attachment_file, self.attachment_filename) + attachment_file.close() + #remove the temp file + os.remove(self.attachment_filename) + self.logger.info("Added attachment successfully.") + except: + self.logger.error("Error while attaching file to ticket.") + return False + + return True + def get_ticket_reported_assets(self, ticket): #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones) return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw)))) @@ -193,12 +264,8 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): if self.is_ticket_resolved(self.jira.issue(ticketid)): self.reopen_ticket(ticketid) - try: - tpl = template(self.template_path, vuln) - except Exception as e: - self.logger.error('Exception updating assets: {}'.format(str(e))) - return 0 - + + #First will do the comparison of assets ticket_obj = self.jira.issue(ticketid) ticket_obj.update() assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips'])))) @@ -211,21 +278,40 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): for asset in difference: if asset in assets: if not added: - added = 'The following assets *have been newly detected*:\n' + added = '\nThe following assets *have been newly detected*:\n' added += '* {}\n'.format(asset) elif asset in ticket_assets: if not removed: - removed= 'The following assets *have been resolved*:\n' + removed= '\nThe following assets *have been resolved*:\n' removed += '* {}\n'.format(asset) comment = added + removed + + #then will check if assets are too many that need to be added as an attachment + attachment_contents = [] + if len(vuln['ips']) > self.max_ips_ticket: + attachment_contents = vuln['ips'] + vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))] + + #fill the ticket description template + try: + tpl = template(self.template_path, vuln) + except Exception as e: + self.logger.error('Exception updating assets: {}'.format(str(e))) + return 0 + #proceed checking if it requires adding as an attachment try: + #update attachment with hosts and delete the old versions + if attachment_contents: + self.clean_old_attachments(ticket_obj) + self.add_content_as_attachment(ticket_obj, attachment_contents) + ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels}) self.logger.info("Ticket {} updated successfully".format(ticketid)) self.add_label(ticketid, 'updated') - except: - self.logger.error("Error while trying up update ticket {}".format(ticketid)) + except Exception as e: + self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e)) return 0 def add_label(self, ticketid, label): From f170dcb05f626cee74dbc7d65b0154f75392a1df Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 12:27:30 +0100 Subject: [PATCH 10/31] reorg resources files --- .gitignore | 1 + docker-compose.v6.yml | 6 +- elk6/vulnwhisperer.ini | 109 ------------------ .../docker-compose_ELK5_unsupported.yml | 0 .../logstash-vulnwhisperer-template.json | 0 .../filebeat}/filebeat.yml | 0 .../1000_vulnWhispererBaseVisuals.json | 0 ...hisperer_ReportingMitigationDashboard.json | 0 ...alysVisuals (required with Dashboard).json | 0 ...eportingMitigationDashboardQualysRisk.json | 0 .../9000_vulnWhisperer_SavedSearch.json | 0 .../logstash}/0001_input_beats.conf | 0 .../logstash}/1000_nessus_process_file.conf | 0 .../logstash}/2000_qualys_web_scans.conf | 0 .../logstash}/3000_openvas.conf | 0 .../logstash}/4000_jira.conf | 0 .../logstash}/9998_input_broker_rabbitmq.conf | 0 .../9998_output_broker_rabbitmq.conf | 0 {elk6 => resources/elk6}/filebeat.yml | 0 {elk6 => resources/elk6}/kibana.json | 0 {elk6 => resources/elk6}/logstash.yml | 0 .../pipeline/1000_nessus_process_file.conf | 0 .../elk6}/pipeline/2000_qualys_web_scans.conf | 0 .../elk6}/pipeline/3000_openvas.conf | 0 .../elk6}/pipeline/4000_jira.conf | 0 25 files changed, 4 insertions(+), 112 deletions(-) delete mode 100644 elk6/vulnwhisperer.ini rename docker-compose.yml => resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml (100%) rename {elasticsearch => resources/elk5-old_compatibility/elasticsearch}/logstash-vulnwhisperer-template.json (100%) rename {filebeat => resources/elk5-old_compatibility/filebeat}/filebeat.yml (100%) rename {kibana => resources/elk5-old_compatibility/kibana}/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json (100%) rename {kibana => resources/elk5-old_compatibility/kibana}/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json (100%) rename {kibana => resources/elk5-old_compatibility/kibana}/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json (100%) rename {kibana => resources/elk5-old_compatibility/kibana}/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json (100%) rename {kibana => resources/elk5-old_compatibility/kibana}/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/0001_input_beats.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/1000_nessus_process_file.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/2000_qualys_web_scans.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/3000_openvas.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/4000_jira.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/9998_input_broker_rabbitmq.conf (100%) rename {logstash => resources/elk5-old_compatibility/logstash}/9998_output_broker_rabbitmq.conf (100%) rename {elk6 => resources/elk6}/filebeat.yml (100%) rename {elk6 => resources/elk6}/kibana.json (100%) rename {elk6 => resources/elk6}/logstash.yml (100%) rename {elk6 => resources/elk6}/pipeline/1000_nessus_process_file.conf (100%) rename {elk6 => resources/elk6}/pipeline/2000_qualys_web_scans.conf (100%) rename {elk6 => resources/elk6}/pipeline/3000_openvas.conf (100%) rename {elk6 => resources/elk6}/pipeline/4000_jira.conf (100%) diff --git a/.gitignore b/.gitignore index ea26da2..b4a878c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ data/ logs/ elk6/vulnwhisperer.ini +resources/elk6/vulnwhisperer.ini configs/frameworks_example.ini # Byte-compiled / optimized / DLL files diff --git a/docker-compose.v6.yml b/docker-compose.v6.yml index 58daac9..e61e1e9 100644 --- a/docker-compose.v6.yml +++ b/docker-compose.v6.yml @@ -44,8 +44,8 @@ services: image: docker.elastic.co/logstash/logstash:6.6.0 container_name: logstash volumes: - - ./elk6/pipeline/:/usr/share/logstash/pipeline - #- ./elk6/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline + #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml - ./data/:/opt/vulnwhisperer/data environment: - xpack.monitoring.enabled=false @@ -65,7 +65,7 @@ services: ] volumes: - ./data/:/opt/vulnwhisperer/data - - ./elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini + - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini network_mode: host volumes: esdata1: diff --git a/elk6/vulnwhisperer.ini b/elk6/vulnwhisperer.ini deleted file mode 100644 index 2b92761..0000000 --- a/elk6/vulnwhisperer.ini +++ /dev/null @@ -1,109 +0,0 @@ -[nessus] -enabled=true -hostname=localhost -port=8834 -username=nessus_username -password=nessus_password -write_path=/opt/vulnwhisperer/data/nessus/ -db_path=/opt/vulnwhisperer/database -trash=false -verbose=true - -[tenable] -enabled=true -hostname=cloud.tenable.com -port=443 -username=tenable.io_username -password=tenable.io_password -write_path=/opt/vulnwhisperer/data/tenable/ -db_path=/opt/VulnWhisperer/data/database -trash=false -verbose=true - -[qualys_web] -#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true -hostname = qualysapi.qg2.apps.qualys.com -username = exampleuser -password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database -verbose=true - -# Set the maximum number of retries each connection should attempt. -#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. -max_retries = 10 -# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. -template_id = 126024 - -[qualys_vuln] -#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true -hostname = qualysapi.qg2.apps.qualys.com -username = exampleuser -password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database -verbose=true - -# Set the maximum number of retries each connection should attempt. -#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. -max_retries = 10 -# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. -template_id = 126024 - -[detectify] -#Reference https://developer.detectify.com/ -enabled = false -hostname = api.detectify.com -#username variable used as apiKey -username = exampleuser -#password variable used as secretKey -password = examplepass -write_path =/opt/vulnwhisperer/data/detectify/ -db_path = /opt/vulnwhisperer/data/database -verbose = true - -[openvas] -enabled = false -hostname = localhost -port = 4000 -username = exampleuser -password = examplepass -write_path=/opt/vulnwhisperer/data/openvas/ -db_path=/opt/vulnwhisperer/data/database -verbose=true - -#[proxy] -; This section is optional. Leave it out if you're not using a proxy. -; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies - -; proxy_protocol set to https, if not specified. -#proxy_url = proxy.mycorp.com - -; proxy_port will override any port specified in proxy_url -#proxy_port = 8080 - -; proxy authentication -#proxy_username = proxyuser -#proxy_password = proxypass - -[jira] -hostname = jira-host -username = username -password = password -write_path = /opt/vulnwhisperer/data/jira/ -db_path = /opt/vulnwhisperer/data/database -verbose = true -dns_resolv = False - -#Sample jira report scan, will automatically be created for existent scans -#[jira.qualys_vuln.test_scan] -#source = qualys_vuln -#scan_name = Test Scan -#jira_project = PROJECT -; if multiple components, separate by "," = None -#components = -; minimum criticality to report (low, medium, high or critical) = None -#min_critical_to_report = high - diff --git a/docker-compose.yml b/resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml similarity index 100% rename from docker-compose.yml rename to resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml diff --git a/elasticsearch/logstash-vulnwhisperer-template.json b/resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json similarity index 100% rename from elasticsearch/logstash-vulnwhisperer-template.json rename to resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json diff --git a/filebeat/filebeat.yml b/resources/elk5-old_compatibility/filebeat/filebeat.yml similarity index 100% rename from filebeat/filebeat.yml rename to resources/elk5-old_compatibility/filebeat/filebeat.yml diff --git a/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json b/resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json similarity index 100% rename from kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json rename to resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json diff --git a/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json b/resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json similarity index 100% rename from kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json rename to resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json diff --git a/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json b/resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json similarity index 100% rename from kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json rename to resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json diff --git a/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json b/resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json similarity index 100% rename from kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json rename to resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json diff --git a/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json b/resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json similarity index 100% rename from kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json rename to resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json diff --git a/logstash/0001_input_beats.conf b/resources/elk5-old_compatibility/logstash/0001_input_beats.conf similarity index 100% rename from logstash/0001_input_beats.conf rename to resources/elk5-old_compatibility/logstash/0001_input_beats.conf diff --git a/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf similarity index 100% rename from logstash/1000_nessus_process_file.conf rename to resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf diff --git a/logstash/2000_qualys_web_scans.conf b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf similarity index 100% rename from logstash/2000_qualys_web_scans.conf rename to resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf diff --git a/logstash/3000_openvas.conf b/resources/elk5-old_compatibility/logstash/3000_openvas.conf similarity index 100% rename from logstash/3000_openvas.conf rename to resources/elk5-old_compatibility/logstash/3000_openvas.conf diff --git a/logstash/4000_jira.conf b/resources/elk5-old_compatibility/logstash/4000_jira.conf similarity index 100% rename from logstash/4000_jira.conf rename to resources/elk5-old_compatibility/logstash/4000_jira.conf diff --git a/logstash/9998_input_broker_rabbitmq.conf b/resources/elk5-old_compatibility/logstash/9998_input_broker_rabbitmq.conf similarity index 100% rename from logstash/9998_input_broker_rabbitmq.conf rename to resources/elk5-old_compatibility/logstash/9998_input_broker_rabbitmq.conf diff --git a/logstash/9998_output_broker_rabbitmq.conf b/resources/elk5-old_compatibility/logstash/9998_output_broker_rabbitmq.conf similarity index 100% rename from logstash/9998_output_broker_rabbitmq.conf rename to resources/elk5-old_compatibility/logstash/9998_output_broker_rabbitmq.conf diff --git a/elk6/filebeat.yml b/resources/elk6/filebeat.yml similarity index 100% rename from elk6/filebeat.yml rename to resources/elk6/filebeat.yml diff --git a/elk6/kibana.json b/resources/elk6/kibana.json similarity index 100% rename from elk6/kibana.json rename to resources/elk6/kibana.json diff --git a/elk6/logstash.yml b/resources/elk6/logstash.yml similarity index 100% rename from elk6/logstash.yml rename to resources/elk6/logstash.yml diff --git a/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf similarity index 100% rename from elk6/pipeline/1000_nessus_process_file.conf rename to resources/elk6/pipeline/1000_nessus_process_file.conf diff --git a/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf similarity index 100% rename from elk6/pipeline/2000_qualys_web_scans.conf rename to resources/elk6/pipeline/2000_qualys_web_scans.conf diff --git a/elk6/pipeline/3000_openvas.conf b/resources/elk6/pipeline/3000_openvas.conf similarity index 100% rename from elk6/pipeline/3000_openvas.conf rename to resources/elk6/pipeline/3000_openvas.conf diff --git a/elk6/pipeline/4000_jira.conf b/resources/elk6/pipeline/4000_jira.conf similarity index 100% rename from elk6/pipeline/4000_jira.conf rename to resources/elk6/pipeline/4000_jira.conf From bdbe31d425e8aa82ddc47f3d67b9540a59cd25b0 Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 12:29:00 +0100 Subject: [PATCH 11/31] resources reorg 2 --- .../elk5-old_compatibility/docker}/1000_nessus_process_file.conf | 0 .../elk5-old_compatibility/docker}/2000_qualys_web_scans.conf | 0 .../elk5-old_compatibility/docker}/3000_openvas.conf | 0 .../elk5-old_compatibility/docker}/4000_jira.conf | 0 {docker => resources/elk5-old_compatibility/docker}/logstash.yml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {docker => resources/elk5-old_compatibility/docker}/1000_nessus_process_file.conf (100%) rename {docker => resources/elk5-old_compatibility/docker}/2000_qualys_web_scans.conf (100%) rename {docker => resources/elk5-old_compatibility/docker}/3000_openvas.conf (100%) rename {docker => resources/elk5-old_compatibility/docker}/4000_jira.conf (100%) rename {docker => resources/elk5-old_compatibility/docker}/logstash.yml (100%) diff --git a/docker/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf similarity index 100% rename from docker/1000_nessus_process_file.conf rename to resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf diff --git a/docker/2000_qualys_web_scans.conf b/resources/elk5-old_compatibility/docker/2000_qualys_web_scans.conf similarity index 100% rename from docker/2000_qualys_web_scans.conf rename to resources/elk5-old_compatibility/docker/2000_qualys_web_scans.conf diff --git a/docker/3000_openvas.conf b/resources/elk5-old_compatibility/docker/3000_openvas.conf similarity index 100% rename from docker/3000_openvas.conf rename to resources/elk5-old_compatibility/docker/3000_openvas.conf diff --git a/docker/4000_jira.conf b/resources/elk5-old_compatibility/docker/4000_jira.conf similarity index 100% rename from docker/4000_jira.conf rename to resources/elk5-old_compatibility/docker/4000_jira.conf diff --git a/docker/logstash.yml b/resources/elk5-old_compatibility/docker/logstash.yml similarity index 100% rename from docker/logstash.yml rename to resources/elk5-old_compatibility/docker/logstash.yml From 05420ddfd0621d8b07c9b7d8bae0742bf0ebcfe8 Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 12:32:32 +0100 Subject: [PATCH 12/31] readding docker-compose credentials template --- resources/elk6/vulnwhisperer.ini | 109 +++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 resources/elk6/vulnwhisperer.ini diff --git a/resources/elk6/vulnwhisperer.ini b/resources/elk6/vulnwhisperer.ini new file mode 100644 index 0000000..2b92761 --- /dev/null +++ b/resources/elk6/vulnwhisperer.ini @@ -0,0 +1,109 @@ +[nessus] +enabled=true +hostname=localhost +port=8834 +username=nessus_username +password=nessus_password +write_path=/opt/vulnwhisperer/data/nessus/ +db_path=/opt/vulnwhisperer/database +trash=false +verbose=true + +[tenable] +enabled=true +hostname=cloud.tenable.com +port=443 +username=tenable.io_username +password=tenable.io_password +write_path=/opt/vulnwhisperer/data/tenable/ +db_path=/opt/VulnWhisperer/data/database +trash=false +verbose=true + +[qualys_web] +#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API +enabled = true +hostname = qualysapi.qg2.apps.qualys.com +username = exampleuser +password = examplepass +write_path=/opt/vulnwhisperer/data/qualys/ +db_path=/opt/vulnwhisperer/data/database +verbose=true + +# Set the maximum number of retries each connection should attempt. +#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. +max_retries = 10 +# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. +template_id = 126024 + +[qualys_vuln] +#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API +enabled = true +hostname = qualysapi.qg2.apps.qualys.com +username = exampleuser +password = examplepass +write_path=/opt/vulnwhisperer/data/qualys/ +db_path=/opt/vulnwhisperer/data/database +verbose=true + +# Set the maximum number of retries each connection should attempt. +#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. +max_retries = 10 +# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. +template_id = 126024 + +[detectify] +#Reference https://developer.detectify.com/ +enabled = false +hostname = api.detectify.com +#username variable used as apiKey +username = exampleuser +#password variable used as secretKey +password = examplepass +write_path =/opt/vulnwhisperer/data/detectify/ +db_path = /opt/vulnwhisperer/data/database +verbose = true + +[openvas] +enabled = false +hostname = localhost +port = 4000 +username = exampleuser +password = examplepass +write_path=/opt/vulnwhisperer/data/openvas/ +db_path=/opt/vulnwhisperer/data/database +verbose=true + +#[proxy] +; This section is optional. Leave it out if you're not using a proxy. +; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies + +; proxy_protocol set to https, if not specified. +#proxy_url = proxy.mycorp.com + +; proxy_port will override any port specified in proxy_url +#proxy_port = 8080 + +; proxy authentication +#proxy_username = proxyuser +#proxy_password = proxypass + +[jira] +hostname = jira-host +username = username +password = password +write_path = /opt/vulnwhisperer/data/jira/ +db_path = /opt/vulnwhisperer/data/database +verbose = true +dns_resolv = False + +#Sample jira report scan, will automatically be created for existent scans +#[jira.qualys_vuln.test_scan] +#source = qualys_vuln +#scan_name = Test Scan +#jira_project = PROJECT +; if multiple components, separate by "," = None +#components = +; minimum criticality to report (low, medium, high or critical) = None +#min_critical_to_report = high + From b36e31566ee196e8fd03da94ebbf1484453f6dc6 Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 22:02:20 +0100 Subject: [PATCH 13/31] fix #142 --- vulnwhisp/vulnwhisp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 287e919..ba30aeb 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -679,6 +679,8 @@ def __init__( if debug: self.logger.setLevel(logging.DEBUG) + + self.directory_check() self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True self.purge = purge From 46ddee391b141b11c30e1354ea7f52affdc47ada Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 25 Feb 2019 22:09:29 +0100 Subject: [PATCH 14/31] confirm openvas 9 works --- configs/frameworks_example.ini | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 77d283c..cdd215c 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -1,5 +1,5 @@ [nessus] -enabled=true +enabled=false hostname=localhost port=8834 username=nessus_username @@ -10,7 +10,7 @@ trash=false verbose=true [tenable] -enabled=true +enabled=false hostname=cloud.tenable.com port=443 username=tenable.io_username @@ -22,7 +22,7 @@ verbose=true [qualys_web] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true +enabled = false hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass @@ -38,7 +38,7 @@ template_id = 126024 [qualys_vuln] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true +enabled = false hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass @@ -65,13 +65,14 @@ db_path = /opt/VulnWhisperer/data/database verbose = true [openvas] -enabled = false -hostname = localhost -port = 4000 -username = exampleuser -password = examplepass -write_path=/opt/VulnWhisperer/data/openvas/ -db_path=/opt/VulnWhisperer/data/database +enabled = true +hostname = 192.168.1.49 +port = 443 +#4000 +username = admin +password = admin +write_path=./data/openvas/ +db_path=./data/database verbose=true #[proxy] From a3da41e4870c3c9dd6e66782364a4645e940e7b1 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 26 Feb 2019 09:59:50 +0100 Subject: [PATCH 15/31] added to readme openvas supported versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d9ffe86..0315a55 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Currently Supports - [X] [Nessus (**v6**/**v7**/**v8**)](https://www.tenable.com/products/nessus/nessus-professional) - [X] [Qualys Web Applications](https://www.qualys.com/apps/web-app-scanning/) - [X] [Qualys Vulnerability Management](https://www.qualys.com/apps/vulnerability-management/) -- [X] [OpenVAS](http://www.openvas.org/) +- [X] [OpenVAS (**v7**/**v8**/**v9**)](http://www.openvas.org/) - [X] [Tenable.io](https://www.tenable.com/products/tenable-io) - [ ] [Detectify](https://detectify.com/) - [ ] [Nexpose](https://www.rapid7.com/products/nexpose/) From 4e94bef2450867d3b8a6bca4b5cf4f5fac5aad64 Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 26 Feb 2019 15:26:14 +0100 Subject: [PATCH 16/31] fix bug not detecting existent label due to string format --- vulnwhisp/reporting/jira_api.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 04afe36..1bf1720 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -317,14 +317,15 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): def add_label(self, ticketid, label): ticket_obj = self.jira.issue(ticketid) - if label not in ticket_obj.fields.labels: - ticket_obj.fields.labels.append(label) + if label not in [x.encode('utf8') for x in ticket_obj.fields.labels]: + ticket_obj.fields.labels.append(label) + + try: + ticket_obj.update(fields={"labels":ticket_obj.fields.labels}) + self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) + except: + self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) - try: - ticket_obj.update(fields={"labels":ticket_obj.fields.labels}) - self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) - except: - self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) return 0 def close_fixed_tickets(self, vulnerabilities): From 623c881928325b5217bc1c0cfbbe045df907fb6c Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 27 Feb 2019 11:27:44 +0100 Subject: [PATCH 17/31] fix jira issue index when comparing created tickets --- vulnwhisp/reporting/jira_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 1bf1720..4ed8e3b 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -151,9 +151,9 @@ def check_vuln_already_exists(self, vuln): #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues self.logger.info("Comparing Vulnerabilities to created tickets") - for index in range(len(self.all_tickets)-1): + for index in range(len(self.all_tickets)): checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index]) - if title == checking_title: + if title.encode('ascii') == checking_title.encode('ascii'): difference = list(set(assets).symmetric_difference(checking_assets)) #to check intersection - set(assets) & set(checking_assets) if difference: From a288f416f7f26b7b4eedb143595bdfabb7aedca8 Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 27 Feb 2019 18:06:16 +0100 Subject: [PATCH 18/31] added label *false positive* for reporting on jira --- vulnwhisp/reporting/jira_api.py | 9 +++++++-- vulnwhisp/reporting/resources/ticket.tpl | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 4ed8e3b..a9dee8f 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -385,6 +385,9 @@ def is_risk_accepted(self, ticket_obj): elif "server_decommission" in labels: self.logger.warn("Ticket {} server decommissioned, will be ignored".format(ticket_obj)) return True + elif "false_positive" in labels: + self.logger.warn("Ticket {} flagged false positive, will be ignored".format(ticket_obj)) + return True self.logger.info("Ticket {} risk has not been accepted".format(ticket_obj)) return False @@ -398,8 +401,10 @@ def reopen_ticket(self, ticketid): try: if self.is_ticket_reopenable(ticket_obj): comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported). - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. + - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. + - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. + - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor. + If you have further doubts, please contact the Security Team.''' error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment) self.logger.info("Ticket {} reopened successfully".format(ticketid)) diff --git a/vulnwhisp/reporting/resources/ticket.tpl b/vulnwhisp/reporting/resources/ticket.tpl index 675a560..dc03b38 100644 --- a/vulnwhisp/reporting/resources/ticket.tpl +++ b/vulnwhisp/reporting/resources/ticket.tpl @@ -30,3 +30,5 @@ Please do not delete or modify the ticket assigned tags or title, as they are us In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. + +If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor. From 86e792f5aafa3b03b547feba0cb3817da9541787 Mon Sep 17 00:00:00 2001 From: Quim Date: Fri, 1 Mar 2019 15:18:49 +0100 Subject: [PATCH 19/31] workaround regarding ignoring ticket updates after risk accepted --- vulnwhisp/reporting/jira_api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index a9dee8f..1039b91 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -262,11 +262,19 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): # correct description will always be in the vulnerability to report, only needed to update description to new one self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid)) - if self.is_ticket_resolved(self.jira.issue(ticketid)): + #for now, if a vulnerability has been accepted ('accepted_risk'), ticket is completely ignored and not updated (no new assets) + + #TODO when vulnerability accepted, create a new ticket with only the non-accepted vulnerable assets + #this would require go through the downloaded tickets, check duplicates/accepted ones, and if so, + #check on their assets to exclude them from the new ticket + risk_accepted = False + ticket_obj = self.jira.issue(ticketid) + if self.is_ticket_resolved(ticket_obj): + if self.is_risk_accepted(ticket_obj): + return 0 self.reopen_ticket(ticketid) #First will do the comparison of assets - ticket_obj = self.jira.issue(ticketid) ticket_obj.update() assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips'])))) difference = list(set(assets).symmetric_difference(ticket_assets)) From 401dfec2c8ba578962ae7faa431d9e049c0351cf Mon Sep 17 00:00:00 2001 From: Quim Date: Mon, 4 Mar 2019 15:10:51 +0100 Subject: [PATCH 20/31] fix #143, added a temporary container to upload through kibana API --- docker-compose.v6.yml | 14 +- resources/elk6/init_kibana.sh | 33 +++ resources/elk6/kibana_APIonly.json | 428 +++++++++++++++++++++++++++++ 3 files changed, 474 insertions(+), 1 deletion(-) create mode 100755 resources/elk6/init_kibana.sh create mode 100755 resources/elk6/kibana_APIonly.json diff --git a/docker-compose.v6.yml b/docker-compose.v6.yml index e61e1e9..b5a833e 100644 --- a/docker-compose.v6.yml +++ b/docker-compose.v6.yml @@ -40,13 +40,24 @@ services: esnet: aliases: - kibana.local + kibana-config: + image: alpine + container_name: kibana-config + volumes: + - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh + - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json + command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json" + networks: + esnet: + aliases: + - kibana-config.local logstash: image: docker.elastic.co/logstash/logstash:6.6.0 container_name: logstash volumes: - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline - #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml - ./data/:/opt/vulnwhisperer/data + #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml environment: - xpack.monitoring.enabled=false depends_on: @@ -64,6 +75,7 @@ services: "/opt/vulnwhisperer/vulnwhisperer.ini" ] volumes: + - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data - ./data/:/opt/vulnwhisperer/data - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini network_mode: host diff --git a/resources/elk6/init_kibana.sh b/resources/elk6/init_kibana.sh new file mode 100755 index 0000000..e9c6075 --- /dev/null +++ b/resources/elk6/init_kibana.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +#kibana_url="localhost:5601" +kibana_url="kibana.local:5601" +add_saved_objects="curl -u elastic:changeme -k -XPOST 'http://"$kibana_url"/api/saved_objects/_bulk_create' -H 'Content-Type: application/json' -H \"kbn-xsrf: true\" -d @" + +#Create all saved objects - including index pattern +saved_objects_file="kibana_APIonly.json" + +#if [ `curl -I localhost:5601/status | head -n1 |cut -d$' ' -f2` -eq '200' ]; then echo "Loading VulnWhisperer Saved Objects"; eval $(echo $add_saved_objects$saved_objects_file); else echo "waiting for kibana"; fi + +until [ "`curl -I "$kibana_url"/status | head -n1 |cut -d$' ' -f2`" == "200" ]; do + curl -I "$kibana_url"/status + echo "Waiting for Kibana" + sleep 5 +done + +echo "Loading VulnWhisperer Saved Objects" +echo $add_saved_objects$saved_objects_file +eval $(echo $add_saved_objects$saved_objects_file) + +#set "*" as default index +#id_default_index="87f3bcc0-8b37-11e8-83be-afaed4786d8c" +#os.system("curl -X POST -H \"Content-Type: application/json\" -H \"kbn-xsrf: true\" -d '{\"value\":\""+id_default_index+"\"}' http://elastic:changeme@"+kibana_url+"kibana/settings/defaultIndex") + +#Create vulnwhisperer index pattern +#index_name = "logstash-vulnwhisperer-*" +#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'") + +#Create jira index pattern, separated for not fill of crap variables the Discover tab by default +#index_name = "logstash-jira-*" +#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'") + diff --git a/resources/elk6/kibana_APIonly.json b/resources/elk6/kibana_APIonly.json new file mode 100755 index 0000000..7a76a29 --- /dev/null +++ b/resources/elk6/kibana_APIonly.json @@ -0,0 +1,428 @@ +[ + { + "id": "AWCUqesWib22Ai8JwW3u", + "type": "dashboard", + "attributes": { + "title": "VulnWhisperer - Risk Mitigation", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":30,\"i\":\"20\",\"w\":8,\"x\":40,\"y\":15},\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":\"20\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":30,\"i\":\"21\",\"w\":12,\"x\":0,\"y\":35},\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":\"21\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":30,\"i\":\"27\",\"w\":12,\"x\":12,\"y\":35},\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":\"27\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":30,\"i\":\"28\",\"w\":8,\"x\":32,\"y\":15},\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":\"28\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":15,\"i\":\"30\",\"w\":8,\"x\":40,\"y\":0},\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":\"30\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":20,\"i\":\"31\",\"w\":8,\"x\":24,\"y\":35},\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":\"31\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"gridData\":{\"h\":10,\"i\":\"37\",\"w\":16,\"x\":16,\"y\":25},\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":\"37\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"sort\":[\"@timestamp\",\"desc\"]},\"gridData\":{\"h\":30,\"i\":\"38\",\"w\":48,\"x\":0,\"y\":65},\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":\"38\",\"type\":\"search\",\"version\":\"6.4.3\"},{\"gridData\":{\"h\":10,\"i\":\"39\",\"w\":16,\"x\":16,\"y\":15},\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":\"39\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"h\":20,\"i\":\"46\",\"w\":16,\"x\":0,\"y\":15},\"id\":\"56f0f5f0-3ebe-11e7-a192-93f36fbd9d05\",\"panelIndex\":\"46\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 50\":\"rgb(247,252,245)\",\"50 - 100\":\"rgb(0,68,27)\"},\"legendOpen\":false}},\"gridData\":{\"h\":15,\"i\":\"47\",\"w\":9,\"x\":30,\"y\":0},\"id\":\"e6b5b920-f77a-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"47\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 10\":\"rgb(255,245,240)\",\"10 - 20\":\"rgb(103,0,13)\"},\"legendOpen\":false}},\"gridData\":{\"h\":15,\"i\":\"48\",\"w\":10,\"x\":0,\"y\":0},\"id\":\"8c9c9430-f77b-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"48\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"0 - 10\":\"#E5AC0E\"},\"defaultColors\":{\"0 - 10\":\"rgb(8,48,107)\"},\"legendOpen\":false}},\"gridData\":{\"h\":15,\"i\":\"50\",\"w\":10,\"x\":20,\"y\":0},\"id\":\"61b43c00-f77b-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"50\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"51\",\"w\":10,\"x\":10,\"y\":0},\"id\":\"c533c120-fe8c-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"51\",\"type\":\"visualization\",\"version\":\"6.4.3\"}]", + "optionsJSON": "{\"darkTheme\":false,\"useMargins\":false}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"match_all\":{}}}}" + } + } + }, + { + "id": "72051530-448e-11e7-a818-f5f80dfc3590", + "type": "dashboard", + "attributes": { + "title": "VulnWhisperer - Reporting", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":20,\"i\":\"5\",\"w\":24,\"x\":0,\"y\":56},\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":\"5\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"gridData\":{\"h\":20,\"i\":\"12\",\"w\":24,\"x\":0,\"y\":36},\"id\":\"8d9592d0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":\"12\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"gridData\":{\"h\":20,\"i\":\"14\",\"w\":24,\"x\":24,\"y\":16},\"id\":\"67d432e0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":\"14\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":20,\"i\":\"15\",\"w\":12,\"x\":36,\"y\":36},\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":\"15\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":20,\"i\":\"20\",\"w\":12,\"x\":24,\"y\":36},\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":\"20\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"gridData\":{\"h\":15,\"i\":\"22\",\"w\":8,\"x\":40,\"y\":0},\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":\"22\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"gridData\":{\"h\":20,\"i\":\"29\",\"w\":24,\"x\":0,\"y\":16},\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":\"29\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 50\":\"rgb(247,252,245)\",\"50 - 100\":\"rgb(0,68,27)\"},\"legendOpen\":false}},\"gridData\":{\"h\":16,\"i\":\"30\",\"w\":10,\"x\":30,\"y\":0},\"id\":\"e6b5b920-f77a-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"30\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"0 - 10\":\"#EAB839\"},\"defaultColors\":{\"0 - 10\":\"rgb(8,48,107)\"},\"legendOpen\":false}},\"gridData\":{\"h\":16,\"i\":\"31\",\"w\":9,\"x\":21,\"y\":0},\"id\":\"61b43c00-f77b-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"31\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"10 - 20\":\"#890F02\"},\"defaultColors\":{\"0 - 10\":\"rgb(255,245,240)\",\"10 - 20\":\"rgb(103,0,13)\"},\"legendOpen\":false}},\"gridData\":{\"h\":16,\"i\":\"32\",\"w\":11,\"x\":0,\"y\":0},\"id\":\"8c9c9430-f77b-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"32\",\"type\":\"visualization\",\"version\":\"6.4.3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"33\",\"w\":10,\"x\":11,\"y\":0},\"id\":\"c533c120-fe8c-11e8-8f42-af2e41422cf8\",\"panelIndex\":\"33\",\"type\":\"visualization\",\"version\":\"6.4.3\"}]", + "optionsJSON": "{\"darkTheme\":false,\"useMargins\":false}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-7d", + "refreshInterval": { + "pause": true, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"match_all\":{}}}}" + } + } + }, + { + "id": "4a6d9090-f66e-11e8-8f42-af2e41422cf8", + "type": "index-pattern", + "attributes": { + "title": "logstash-vulnwhisperer-*", + "timeFieldName": "@timestamp", + "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"IpPort\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"asset\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"asset.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"assign_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cvss\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"description.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"destination_geo.ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"destination_geo.latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"destination_geo.location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"destination_geo.longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"history_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"history_id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"last_updated\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message_error\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"opcode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"path.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"plugin_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"plugin_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"plugin_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"plugin_output\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"plugin_output.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"record_number\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"risk\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"risk_number\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"risk_number.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"risk_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"risk_score_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"risk_score_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"scan_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"scan_id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"scan_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"scan_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"see_also\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"solution\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_geo.ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_geo.latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_geo.location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_geo.longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"synopsis\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tags.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"scan_fingerprint\",\"type\":\"string\",\"count\":1,\"scripted\":true,\"script\":\"doc['asset.keyword']+'_'+doc['plugin_id']\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]", + "fieldFormatMap": "{\"plugin_id\":{\"id\":\"number\",\"params\":{\"pattern\":\"00.[000]\"}}}" + } + }, + { + "id": "159d2500-f773-11e8-8f42-af2e41422cf8", + "type": "search", + "attributes": { + "title": "VulnWhisperer - High Risk", + "description": "", + "hits": 0, + "columns": [ + "host", + "risk", + "risk_score", + "cve", + "plugin_name", + "solution", + "plugin_output" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"type\":\"phrase\",\"key\":\"risk\",\"value\":\"High\",\"params\":{\"query\":\"High\",\"type\":\"phrase\"},\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"risk\":{\"query\":\"High\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"highlightAll\":true,\"version\":true}" + } + } + }, + { + "id": "54648700-3f74-11e7-852e-69207a3d0726", + "type": "search", + "attributes": { + "title": "VulnWhisperer - Saved Search", + "description": "", + "hits": 0, + "columns": [ + "host", + "risk", + "risk_score", + "cve", + "plugin_name", + "solution", + "plugin_output" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + } + } + }, + { + "id": "41a7e430-fdb5-11e8-8f42-af2e41422cf8", + "type": "search", + "attributes": { + "title": "VulnWhisperer - Compliance", + "description": "", + "hits": 0, + "columns": [ + "plugin_id", + "cve", + "cvss", + "risk", + "asset", + "protocol", + "port", + "plugin_name", + "synopsis", + "description", + "solution", + "see_also", + "plugin_output" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + } + }, + { + "id": "465c5820-8977-11e7-857e-e1d56b17746d", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Critical Assets", + "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"colors\":{\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\",\"7 - 9\":\"#EF843C\",\"3 - 7\":\"#EAB839\",\"0 - 3\":\"#7EB26D\"},\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"index\":\"logstash-vulnwhisperer-*\",\"negate\":false,\"disabled\":false,\"alias\":\"Critical Asset\",\"type\":\"phrase\",\"key\":\"tags\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}" + } + } + }, + { + "id": "56f0f5f0-3ebe-11e7-a192-93f36fbd9d05", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer-RiskOverTime", + "visState": "{\"title\":\"VulnWhisperer-RiskOverTime\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "5093c620-44e9-11e7-8014-ede06a7e69f8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Mitigation Readme", + "visState": "{\"title\":\"VulnWhisperer - Mitigation Readme\",\"type\":\"markdown\",\"params\":{\"markdown\":\"** Legend **\\n\\n* [Common Vulnerability Scoring System (CVSS)](https://nvd.nist.gov/vuln-metrics/cvss) is the NIST vulnerability scoring system\\n* Risk Number is residual risk score calculated from CVSS, which is adjusted to be specific to Heartland which accounts for services not in use such as Java and Flash\\n* Vulnerabilities by Tag are systems tagged with HIPAA and PCI identification.\\n\\n\\n** Workflow **\\n* Select 10.0 under Risk Number to identify Critical Vulnerabilities. \\n* For more information about a CVE, scroll down and click the CVE link.\\n* To filter by tags, use one of the following filters:\\n** tags:has_hipaa_data, tags:pci_asset, tags:hipaa_asset, tags:critical_asset**\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "471a3580-3f6b-11e7-88e7-df1abe6547fb", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Vulnerabilities by Tag", + "visState": "{\"title\":\"VulnWhisperer - Vulnerabilities by Tag\",\"type\":\"table\",\"params\":{\"perPage\":3,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:has_hipaa_data\",\"analyze_wildcard\":true}}},\"label\":\"Systems with HIPAA data\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:pci_asset\",\"analyze_wildcard\":true}}},\"label\":\"PCI Systems\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:hipaa_asset\",\"analyze_wildcard\":true}}},\"label\":\"HIPAA Systems\"}]}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "1de9e550-3df1-11e7-a44e-c79ca8efb780", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer-Description", + "visState": "{\"title\":\"VulnWhisperer-Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"description.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Description\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "fb6eb020-49ab-11e7-8f8c-57ad64ec48a6", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Critical Risk Score for Tagged Assets", + "visState": "{\"title\":\"VulnWhisperer - Critical Risk Score for Tagged Assets\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Systems\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "479deab0-8a39-11e7-a58a-9bfcb3761a3d", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - TL - TaggedAssetsPluginNames", + "visState": "{\"title\":\"VulnWhisperer - TL - TaggedAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset OR tags:hipaa_asset OR tags:pci_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "13c7d4e0-3df3-11e7-a44e-c79ca8efb780", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer-Solution", + "visState": "{\n \"title\": \"VulnWhisperer-Solution\",\n \"type\": \"table\",\n \"params\": {\n \"perPage\": 10,\n \"showMeticsAtAllLevels\": false,\n \"showPartialRows\": false,\n \"showTotal\": false,\n \"sort\": {\n \"columnIndex\": null,\n \"direction\": null\n },\n \"totalFunc\": \"sum\"\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"bucket\",\n \"params\": {\n \"field\": \"solution\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"customLabel\": \"Solution\"\n }\n }\n ],\n \"listeners\": {}\n}", + "uiStateJSON": "{\n \"vis\": {\n \"params\": {\n \"sort\": {\n \"columnIndex\": null,\n \"direction\": null\n }\n }\n }\n}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\n \"query\": {\n \"query\": {\n \"query_string\": {\n \"analyze_wildcard\": true,\n \"query\": \"*\"\n }\n },\n \"language\": \"lucene\"\n },\n \"filter\": []\n}" + } + } + }, + { + "id": "e6b5b920-f77a-11e8-8f42-af2e41422cf8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Risk: Low", + "visState": "{\"title\":\"VulnWhisperer - Risk: Low\",\"type\":\"gauge\",\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":true,\"isDisplayWarning\":false,\"gauge\":{\"verticalSplit\":false,\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Greens\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":50},{\"from\":50,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"#eee\",\"bgColor\":false,\"subText\":\"\",\"fontSize\":60,\"labelColor\":true}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":\"risk:Low\"},\"label\":\"Low Risk\"}]}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 50\":\"rgb(247,252,245)\",\"50 - 100\":\"rgb(0,68,27)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "61b43c00-f77b-11e8-8f42-af2e41422cf8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Risk: Medium", + "visState": "{\"title\":\"VulnWhisperer - Risk: Medium\",\"type\":\"gauge\",\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":true,\"isDisplayWarning\":false,\"gauge\":{\"verticalSplit\":false,\"extendRange\":false,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Blues\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":10}],\"invertColors\":true,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"#eee\",\"bgColor\":false,\"subText\":\"\",\"fontSize\":60,\"labelColor\":true}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":\"risk:Medium\"},\"label\":\"Medium Risk\"}]}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10\":\"rgb(8,48,107)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "f9b68640-fda5-11e8-8f42-af2e41422cf8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - AggTest", + "visState": "{\"title\":\"VulnWhisperer - AggTest\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"asset.keyword\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_id\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "2f979030-44b9-11e7-a818-f5f80dfc3590", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - ScanBarChart", + "visState": "{\"title\":\"VulnWhisperer - ScanBarChart\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"legendPosition\":\"right\",\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"times\":[],\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of scan_fingerprint\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Unique count of scan_fingerprint\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Scan Name\"}}]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "8c9c9430-f77b-11e8-8f42-af2e41422cf8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Risk: Critical", + "visState": "{\"title\":\"VulnWhisperer - Risk: Critical\",\"type\":\"gauge\",\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":true,\"isDisplayWarning\":false,\"gauge\":{\"verticalSplit\":false,\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Reds\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":10},{\"from\":10,\"to\":20}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"#eee\",\"bgColor\":false,\"subText\":\"\",\"fontSize\":60,\"labelColor\":true}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":\"risk:Critical\"},\"label\":\"Critical Risk\"}]}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10\":\"rgb(255,245,240)\",\"10 - 20\":\"rgb(103,0,13)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + } + }, + { + "id": "c533c120-fe8c-11e8-8f42-af2e41422cf8", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Risk: High", + "visState": "{\"title\":\"VulnWhisperer - Risk: High\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Reds\",\"gaugeColorMode\":\"None\",\"colorsRange\":[{\"from\":1,\"to\":5},{\"from\":5,\"to\":19999}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"meter\",\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\",\"customLabel\":\"Risk: High\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":\"risk:High\"},\"label\":\"risk: High\"}]}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"1 - 5\":\"rgb(255,245,240)\",\"5 - 19999\":\"rgb(103,0,13)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "852816e0-3eb1-11e7-90cb-918f9cb01e3d", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer-CVSS", + "visState": "{\"title\":\"VulnWhisperer-CVSS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"},\"totalFunc\":\"sum\",\"type\":\"table\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\",\"customLabel\":\"Unique Findings\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"cvss\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"CVSS Score\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"asset.keyword\",\"customLabel\":\"# of Assets\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "35b6d320-3f7f-11e7-bd24-6903e3283192", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Residual Risk", + "visState": "{\"title\":\"VulnWhisperer - Residual Risk\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk_score\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Risk Number\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "de1a5f40-3f85-11e7-97f9-3777d794626d", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - ScanName", + "visState": "{\"title\":\"VulnWhisperer - ScanName\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"scan_name.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Scan Name\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "995e2280-3df3-11e7-a44e-c79ca8efb780", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer-Asset", + "visState": "{\"title\":\"VulnWhisperer-Asset\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\",\"customLabel\":\"Findings\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"asset.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Asset\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "297df800-3f7e-11e7-bd24-6903e3283192", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - Plugin Name", + "visState": "{\"title\":\"VulnWhisperer - Plugin Name\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"scan_fingerprint\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Plugin Name\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"4a6d9090-f66e-11e8-8f42-af2e41422cf8\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "67d432e0-44ec-11e7-a05f-d9719b331a27", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - TL-Critical Risk", + "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "id": "8d9592d0-44ec-11e7-a05f-d9719b331a27", + "type": "visualization", + "attributes": { + "title": "VulnWhisperer - TL-High Risk", + "visState": "{\"title\":\"VulnWhisperer - TL-High Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" + } + } + } + ] From e7bd4d2a55c3b300be7ee0ad04544319c1515e0b Mon Sep 17 00:00:00 2001 From: Quim Date: Fri, 15 Mar 2019 12:03:02 +0100 Subject: [PATCH 21/31] deleting dependency and pulling qualysapi official library, vulnwhisperer compatible --- .gitmodules | 3 --- deps/qualysapi | 1 - requirements.txt | 1 + 3 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 deps/qualysapi diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e6bb589..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "qualysapi"] - path = deps/qualysapi - url = https://github.com/austin-taylor/qualysapi.git diff --git a/deps/qualysapi b/deps/qualysapi deleted file mode 160000 index 42c3b43..0000000 --- a/deps/qualysapi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 42c3b43ac1e6657be32f7a11d211e3157af4143b diff --git a/requirements.txt b/requirements.txt index 9ecc043..a49e39d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ bs4 jira bottle coloredlogs +qualysapi>=5.1.0 From 936c4a3e1bc510827adad43d3e46c7bed58de3db Mon Sep 17 00:00:00 2001 From: Quim Date: Tue, 19 Mar 2019 12:58:38 +0100 Subject: [PATCH 22/31] added automatic jira server_decommission label removal after x time --- vulnwhisp/reporting/jira_api.py | 94 ++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 1039b91..8a083d7 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -9,7 +9,7 @@ import re class JiraAPI(object): - def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12): + def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3): self.logger = logging.getLogger('JiraAPI') if debug: self.logger.setLevel(logging.DEBUG) @@ -23,11 +23,8 @@ def __init__(self, hostname=None, username=None, password=None, path="", debug=F self.all_tickets = [] self.JIRA_REOPEN_ISSUE = "Reopen Issue" self.JIRA_CLOSE_ISSUE = "Close Issue" - self.max_time_tracking = max_time_window #in months - # self.JIRA_RESOLUTION_OBSOLETE = "Obsolete" self.JIRA_RESOLUTION_FIXED = "Fixed" - self.clean_obsolete = clean_obsolete self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl' self.max_ips_ticket = 30 self.attachment_filename = "vulnerable_assets.txt" @@ -35,6 +32,20 @@ def __init__(self, hostname=None, username=None, password=None, path="", debug=F self.download_tickets(path) else: self.logger.warn("No local path specified, skipping Jira ticket download.") + self.max_time_tracking = max_time_window #in months + self.max_decommission_time = decommission_time_window #in months + # [HIGIENE] close tickets older than 12 months as obsolete (max_time_window defined) + if clean_obsolete: + self.close_obsolete_tickets() + # deletes the tag "server_decommission" from those tickets closed <=3 months ago + self.decommission_cleanup() + + self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported). + - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. + - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. + - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor. + + If you have further doubts, please contact the Security Team.''' def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents = []): labels = ['vulnerability_management'] @@ -88,11 +99,6 @@ def sync(self, vulnerabilities, project, components=[]): #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references] self.logger.info("JIRA Sync started") - # [HIGIENE] close tickets older than 12 months as obsolete - # Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management' - if self.clean_obsolete: - self.close_obsolete_tickets() - for vuln in vulnerabilities: # JIRA doesn't allow labels with spaces, so making sure that the scan_name doesn't have spaces # if it has, they will be replaced by "_" @@ -107,7 +113,7 @@ def sync(self, vulnerabilities, project, components=[]): if exists: # If ticket "resolved" -> reopen, as vulnerability is still existent - self.reopen_ticket(ticketid) + self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment) self.add_label(ticketid, vuln['risk']) continue elif to_update: @@ -251,7 +257,6 @@ def get_resolution_time(self, ticket): start = datetime(created[0],created[1],created[2],created[3],created[4],created[5]) end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5]) - return (end-start).days else: self.logger.error("Ticket {ticket} is not resolved, can't calculate resolution time".format(ticket=ticket)) @@ -272,7 +277,7 @@ def ticket_update_assets(self, vuln, ticketid, ticket_assets): if self.is_ticket_resolved(ticket_obj): if self.is_risk_accepted(ticket_obj): return 0 - self.reopen_ticket(ticketid) + self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment) #First will do the comparison of assets ticket_obj.update() @@ -336,8 +341,27 @@ def add_label(self, ticketid, label): return 0 + def remove_label(self, ticketid, label): + ticket_obj = self.jira.issue(ticketid) + + if label in [x.encode('utf8') for x in ticket_obj.fields.labels]: + ticket_obj.fields.labels.remove(label) + + try: + ticket_obj.update(fields={"labels":ticket_obj.fields.labels}) + self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid)) + except: + self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) + else: + self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid)) + + return 0 + def close_fixed_tickets(self, vulnerabilities): - # close tickets which vulnerabilities have been resolved and are still open + ''' + Close tickets which vulnerabilities have been resolved and are still open. + Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management' + ''' found_vulns = [] for vuln in vulnerabilities: found_vulns.append(vuln['title']) @@ -399,24 +423,19 @@ def is_risk_accepted(self, ticket_obj): self.logger.info("Ticket {} risk has not been accepted".format(ticket_obj)) return False - def reopen_ticket(self, ticketid): + def reopen_ticket(self, ticketid, ignore_labels=False, comment=""): self.logger.debug("Ticket {} exists, REOPEN requested".format(ticketid)) # this will reopen a ticket by ticketid ticket_obj = self.jira.issue(ticketid) if self.is_ticket_resolved(ticket_obj): - if not self.is_risk_accepted(ticket_obj): + if (not self.is_risk_accepted(ticket_obj) or ignore_labels): try: if self.is_ticket_reopenable(ticket_obj): - comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported). - - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it. - - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it. - - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor. - - If you have further doubts, please contact the Security Team.''' error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment) self.logger.info("Ticket {} reopened successfully".format(ticketid)) - self.add_label(ticketid, 'reopened') + if not ignore_labels: + self.add_label(ticketid, 'reopened') return 1 except Exception as e: # continue with ticket data so that a new ticket is created in place of the "lost" one @@ -449,8 +468,8 @@ def close_obsolete_tickets(self): jql = "labels=vulnerability_management AND created Date: Tue, 19 Mar 2019 15:19:27 +0100 Subject: [PATCH 23/31] fix bug --- vulnwhisp/reporting/jira_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 8a083d7..a209290 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -28,11 +28,11 @@ def __init__(self, hostname=None, username=None, password=None, path="", debug=F self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl' self.max_ips_ticket = 30 self.attachment_filename = "vulnerable_assets.txt" + self.max_time_tracking = max_time_window #in months if path: self.download_tickets(path) else: self.logger.warn("No local path specified, skipping Jira ticket download.") - self.max_time_tracking = max_time_window #in months self.max_decommission_time = decommission_time_window #in months # [HIGIENE] close tickets older than 12 months as obsolete (max_time_window defined) if clean_obsolete: From 70e1d7703f65ef95a9e4bc43d8148a5b51420032 Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 20 Mar 2019 08:33:52 +0100 Subject: [PATCH 24/31] fix missing section specification on qualys was connector #156 --- vulnwhisp/frameworks/qualys_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 74b5122..7212e75 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -43,7 +43,7 @@ def __init__(self, config=None): self.logger.error('Could not connect to Qualys: {}'.format(str(e))) self.headers = { "content-type": "text/xml"} - self.config_parse = qcconf.QualysConnectConfig(config) + self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web') try: self.template_id = self.config_parse.get_template_id() except: From 9d52596be94ce47d99f50544fa1b13b3ca7e5de4 Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 20 Mar 2019 08:49:36 +0100 Subject: [PATCH 25/31] fix xml encoding issue #156 --- vulnwhisp/frameworks/qualys_web.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 7212e75..1e7fc1c 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -74,7 +74,7 @@ def get_was_scan_count(self, status): E.filters( E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)))) xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters) - root = objectify.fromstring(xml_output) + root = objectify.fromstring(xml_output.encode('utf-8')) return root.count.text def get_reports(self): @@ -127,17 +127,21 @@ def get_all_scans(self, limit=1000, offset=1, status='FINISHED'): qualys_api_limit = limit dataframes = [] _records = [] - total = int(self.get_was_scan_count(status=status)) - self.logger.info('Retrieving information for {} scans'.format(total)) - for i in range(0, total): - if i % limit == 0: - if (total - i) < limit: - qualys_api_limit = total - i - self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit), str(i + 1)))) - scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status) - _records.append(scan_info) - self.logger.debug('Converting XML to DataFrame') - dataframes = [self.xml_parser(xml) for xml in _records] + try: + total = int(self.get_was_scan_count(status=status)) + self.logger.error('Already have WAS scan count') + self.logger.info('Retrieving information for {} scans'.format(total)) + for i in range(0, total): + if i % limit == 0: + if (total - i) < limit: + qualys_api_limit = total - i + self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit), str(i + 1)))) + scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status) + _records.append(scan_info) + self.logger.debug('Converting XML to DataFrame') + dataframes = [self.xml_parser(xml) for xml in _records] + except Exception as e: + self.logger.error("Couldn't process all scans: {}".format(e)) return pd.concat(dataframes, axis=0).reset_index().drop('index', axis=1) From a4420b7df85c6e30ca86a3deb9303035ec155ea3 Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 20 Mar 2019 09:11:18 +0100 Subject: [PATCH 26/31] reverse unintended change on frameworks_example.ini --- configs/frameworks_example.ini | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index cdd215c..77d283c 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -1,5 +1,5 @@ [nessus] -enabled=false +enabled=true hostname=localhost port=8834 username=nessus_username @@ -10,7 +10,7 @@ trash=false verbose=true [tenable] -enabled=false +enabled=true hostname=cloud.tenable.com port=443 username=tenable.io_username @@ -22,7 +22,7 @@ verbose=true [qualys_web] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = false +enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass @@ -38,7 +38,7 @@ template_id = 126024 [qualys_vuln] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = false +enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass @@ -65,14 +65,13 @@ db_path = /opt/VulnWhisperer/data/database verbose = true [openvas] -enabled = true -hostname = 192.168.1.49 -port = 443 -#4000 -username = admin -password = admin -write_path=./data/openvas/ -db_path=./data/database +enabled = false +hostname = localhost +port = 4000 +username = exampleuser +password = examplepass +write_path=/opt/VulnWhisperer/data/openvas/ +db_path=/opt/VulnWhisperer/data/database verbose=true #[proxy] From 47df1ee538152d079ec9588591e2f38d6928949a Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 20 Mar 2019 10:55:54 +0100 Subject: [PATCH 27/31] typo --- vulnwhisp/reporting/jira_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index a209290..a13bf9e 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -523,7 +523,7 @@ def decommission_cleanup(self): jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time) decommissioned_tickets = self.jira.search_issues(jql, maxResults=0) - comment = '''This ticket is having deleted the *server_decommission* tag deleted, as it is more than {} months old and is expected to already have been decommissioned. + comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned. If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time) for ticket in decommissioned_tickets: From 843aac6a83fbd74c0520da1473aef76dddeb1171 Mon Sep 17 00:00:00 2001 From: Quim Date: Wed, 20 Mar 2019 16:37:50 +0100 Subject: [PATCH 28/31] fixing issue with new vulns of already risk accepted issues not being reported anymore; now, new ticket is raised, excluding all the assets that have been previously considered risk accepted in another ticket --- vulnwhisp/reporting/jira_api.py | 125 +++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 36 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index a13bf9e..e58a5e4 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -21,6 +21,7 @@ def __init__(self, hostname=None, username=None, password=None, path="", debug=F self.jira = JIRA(options={'server': hostname}, basic_auth=(self.username, self.password)) self.logger.info("Created vjira service for {}".format(hostname)) self.all_tickets = [] + self.excluded_tickets = [] self.JIRA_REOPEN_ISSUE = "Reopen Issue" self.JIRA_CLOSE_ISSUE = "Close Issue" self.JIRA_RESOLUTION_OBSOLETE = "Obsolete" @@ -52,7 +53,7 @@ def create_ticket(self, title, desc, project="IS", components=[], tags=[], attac for tag in tags: labels.append(str(tag)) - self.logger.info("creating ticket for project {} title: {}".format(project, title[:20])) + self.logger.info("Creating ticket for project {} title: {}".format(project, title[:20])) self.logger.debug("project {} has a component requirement: {}".format(project, components)) project_obj = self.jira.project(project) components_ticket = [] @@ -105,40 +106,87 @@ def sync(self, vulnerabilities, project, components=[]): if " " in vuln['scan_name']: vuln['scan_name'] = "_".join(vuln['scan_name'].split(" ")) - exists = False - to_update = False - ticketid = "" - ticket_assets = [] - exists, to_update, ticketid, ticket_assets = self.check_vuln_already_exists(vuln) - - if exists: - # If ticket "resolved" -> reopen, as vulnerability is still existent - self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment) - self.add_label(ticketid, vuln['risk']) - continue - elif to_update: - self.ticket_update_assets(vuln, ticketid, ticket_assets) - self.add_label(ticketid, vuln['risk']) - continue - attachment_contents = [] - # if assets >30, add as attachment - # create local text file with assets, attach it to ticket - if len(vuln['ips']) > self.max_ips_ticket: - attachment_contents = vuln['ips'] - vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))] - try: - tpl = template(self.template_path, vuln) - except Exception as e: - self.logger.error('Exception templating: {}'.format(str(e))) - return 0 - self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents) + # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission* + vuln = self.exclude_accepted_assets(vuln) + + # make sure after exclusion of risk_accepted assets there are still assets + if vuln['ips']: + exists = False + to_update = False + ticketid = "" + ticket_assets = [] + exists, to_update, ticketid, ticket_assets = self.check_vuln_already_exists(vuln) + + if exists: + # If ticket "resolved" -> reopen, as vulnerability is still existent + self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment) + self.add_label(ticketid, vuln['risk']) + continue + elif to_update: + self.ticket_update_assets(vuln, ticketid, ticket_assets) + self.add_label(ticketid, vuln['risk']) + continue + attachment_contents = [] + # if assets >30, add as attachment + # create local text file with assets, attach it to ticket + if len(vuln['ips']) > self.max_ips_ticket: + attachment_contents = vuln['ips'] + vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))] + try: + tpl = template(self.template_path, vuln) + except Exception as e: + self.logger.error('Exception templating: {}'.format(str(e))) + return 0 + self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents) + else: + self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket") self.close_fixed_tickets(vulnerabilities) # we reinitialize so the next sync redoes the query with their specific variables self.all_tickets = [] + self.excluded_tickets = [] return True + + def exclude_accepted_assets(self, vuln): + # we want to check JIRA tickets with risk_accepted/server_decommission or false_positive labels sharing the same source + # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable + labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] + + if not self.excluded_tickets: + jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking) + self.excluded_tickets = self.jira.search_issues(jql, maxResults=0) + + title = vuln['title'] + #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists + #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues + self.logger.info("Comparing vulnerability to risk_accepted tickets") + assets_to_exclude = [] + tickets_excluded_assets = [] + for index in range(len(self.excluded_tickets)): + checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index]) + if title.encode('ascii') == checking_title.encode('ascii'): + if checking_assets: + #checking_assets is a list, we add to our full list for later delete all assets + assets_to_exclude+=checking_assets + tickets_excluded_assets.append(checking_ticketid) + + if assets_to_exclude: + self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets))) + #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial + for exclusion in assets_to_exclude: + for asset in vuln['ips']: + if exclusion in asset: + #self.logger.error("Assets before deleting risk_accepted assets: {}".format(vuln['ips'])) + self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(asset,title)) + vuln['ips'].remove(asset) + + return vuln def check_vuln_already_exists(self, vuln): + ''' + This function compares a vulnerability with a collection of tickets. + Returns [exists (bool), is equal (bool), ticketid (str), assets (array)] + ''' # we need to return if the vulnerability has already been reported and the ID of the ticket for further processing #function returns array [duplicated(bool), update(bool), ticketid, ticket_assets] title = vuln['title'] @@ -159,7 +207,8 @@ def check_vuln_already_exists(self, vuln): self.logger.info("Comparing Vulnerabilities to created tickets") for index in range(len(self.all_tickets)): checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index]) - if title.encode('ascii') == checking_title.encode('ascii'): + # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets + if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): difference = list(set(assets).symmetric_difference(checking_assets)) #to check intersection - set(assets) & set(checking_assets) if difference: @@ -173,27 +222,31 @@ def check_vuln_already_exists(self, vuln): def ticket_get_unique_fields(self, ticket): title = ticket.raw.get('fields', {}).get('summary').encode("ascii").strip() ticketid = ticket.key.encode("ascii") + assets = [] try: affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0] assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section))) - + + except Exception as e: + self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e)) + assets = [] + + try: if not assets: #check if attachment, if so, get assets from attachment affected_assets_section = self.check_ips_attachment(ticket) if affected_assets_section: assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section))) - except Exception as e: - self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e)) - assets = [] - + self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e)) + return ticketid, title, assets def check_ips_attachment(self, ticket): affected_assets_section = [] try: - fields = self.jira.issue(ticket.key).raw.get('fields') - attachments = fields.get('attachment') + fields = self.jira.issue(ticket.key).raw.get('fields', {}) + attachments = fields.get('attachment', {}) affected_assets_section = "" #we will make sure we get the latest version of the file latest = '' From a4b1b9cdd423a79e71b671d8f6dafd1b27b4aab2 Mon Sep 17 00:00:00 2001 From: Quim Date: Thu, 21 Mar 2019 15:52:18 +0100 Subject: [PATCH 29/31] fixed issue where, asset after a removed one, was ignored due to python listing --- vulnwhisp/reporting/jira_api.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index e58a5e4..12b3360 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -171,14 +171,18 @@ def exclude_accepted_assets(self, vuln): tickets_excluded_assets.append(checking_ticketid) if assets_to_exclude: + assets_to_remove = [] self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets))) + self.logger.debug("Original assets: {}".format(vuln['ips'])) #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial for exclusion in assets_to_exclude: - for asset in vuln['ips']: - if exclusion in asset: - #self.logger.error("Assets before deleting risk_accepted assets: {}".format(vuln['ips'])) - self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(asset,title)) - vuln['ips'].remove(asset) + # for efficiency, we walk the backwards the array of ips from the scanners, as we will be popping out the matches + # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one) + for index in range(len(vuln['ips']))[::-1]: + if exclusion == vuln['ips'][index].split(" - ")[0]: + self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title)) + vuln['ips'].pop(index) + self.logger.debug("Modified assets: {}".format(vuln['ips'])) return vuln From 97e4f073bfc371a5b35bbeb6ced95f6cd5970cd6 Mon Sep 17 00:00:00 2001 From: Quim Date: Fri, 22 Mar 2019 10:38:55 +0100 Subject: [PATCH 30/31] added logging to file --- bin/vuln_whisperer | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index d94b5ce..4621c23 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -41,7 +41,13 @@ def main(): stream=sys.stdout, level=logging.DEBUG if args.debug else logging.INFO ) - logger = logging.getLogger(name='main') + logger = logging.getLogger() + # we set up the logger to log as well to file + fh = logging.FileHandler('vulnwhisperer.log') + fh.setLevel(logging.DEBUG if args.debug else logging.INFO) + fh.setFormatter(logging.Formatter("%(asctime)s - %(name)s:%(levelname)s:%(message)s", "%Y-%m-%d %H:%M:%S")) + logger.addHandler(fh) + if args.fancy: import coloredlogs coloredlogs.install(level='DEBUG' if args.debug else 'INFO') @@ -68,6 +74,7 @@ def main(): vw.whisper_vulnerabilities() # TODO: fix this to NOT be exit 1 unless in error + close_logging_handlers() sys.exit(1) else: @@ -82,6 +89,7 @@ def main(): vw.whisper_vulnerabilities() # TODO: fix this to NOT be exit 1 unless in error + close_logging_handlers() sys.exit(1) except Exception as e: @@ -90,8 +98,15 @@ def main(): logger.error('{}'.format(str(e))) print('ERROR: {error}'.format(error=e)) # TODO: fix this to NOT be exit 2 unless in error + close_logging_handlers() sys.exit(2) + close_logging_handlers() + +def close_logging_handlers(): + for handler in logger.handlers: + handler.close() + logger.removeFilter(handler) if __name__ == '__main__': main() From 3601ace5e15a37cf823a7d2be19b520e9b5c1b12 Mon Sep 17 00:00:00 2001 From: Quim Date: Fri, 22 Mar 2019 10:42:30 +0100 Subject: [PATCH 31/31] improved file logging format --- bin/vuln_whisperer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 4621c23..cf10616 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -45,7 +45,7 @@ def main(): # we set up the logger to log as well to file fh = logging.FileHandler('vulnwhisperer.log') fh.setLevel(logging.DEBUG if args.debug else logging.INFO) - fh.setFormatter(logging.Formatter("%(asctime)s - %(name)s:%(levelname)s:%(message)s", "%Y-%m-%d %H:%M:%S")) + fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S")) logger.addHandler(fh) if args.fancy: