diff --git a/doc/conf.py b/doc/conf.py index ae3f0ee62..e788f6364 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -50,9 +50,9 @@ master_doc = 'index' # General information about the project. -project = u'HubbleStack' -copyright = u'2018, Colton Myers, Christer Edwards' -author = u'Colton Myers, Christer Edwards' +project = 'HubbleStack' +copyright = '2018, Colton Myers, Christer Edwards' +author = 'Colton Myers, Christer Edwards' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -140,8 +140,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'HubbleStack.tex', u'HubbleStack Documentation', - u'Colton Myers, Christer Edwards', 'manual'), + (master_doc, 'HubbleStack.tex', 'HubbleStack Documentation', + 'Colton Myers, Christer Edwards', 'manual'), ] @@ -150,7 +150,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'hubblestack', u'HubbleStack Documentation', + (master_doc, 'hubblestack', 'HubbleStack Documentation', [author], 1) ] @@ -161,7 +161,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'HubbleStack', u'HubbleStack Documentation', + (master_doc, 'HubbleStack', 'HubbleStack Documentation', author, 'HubbleStack', 'One line description of project.', 'Miscellaneous'), ] diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index ae5c3e4e4..c529a5500 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -3,8 +3,6 @@ Main entry point for the hubble daemon """ -from __future__ import print_function - # import lockfile import argparse import copy @@ -307,7 +305,7 @@ def schedule(): schedule_config = __opts__.get('schedule', {}) if 'user_schedule' in __opts__ and isinstance(__opts__['user_schedule'], dict): schedule_config.update(__opts__['user_schedule']) - for jobname, jobdata in schedule_config.iteritems(): + for jobname, jobdata in schedule_config.items(): # Error handling galore if not jobdata or not isinstance(jobdata, dict): log.error('Scheduled job %s does not have valid data', jobname) @@ -808,7 +806,7 @@ def emit_to_syslog(grains_to_emit): for grain in grains_to_emit: if grain in __grains__: if bool(__grains__[grain]) and isinstance(__grains__[grain], dict): - for key, value in __grains__[grain].iteritems(): + for key, value in __grains__[grain].items(): syslog_list.append('{0}={1}'.format(key, value)) else: syslog_list.append('{0}={1}'.format(grain, __grains__[grain])) diff --git a/hubblestack/extmods/audit/grep.py b/hubblestack/extmods/audit/grep.py index 9c9e40e8c..f993f3371 100644 --- a/hubblestack/extmods/audit/grep.py +++ b/hubblestack/extmods/audit/grep.py @@ -42,7 +42,7 @@ True/False. Whether to use multiline flag for regex matching with match_output_regex set to True. Defaults to True. """ -from __future__ import absolute_import + import logging import os diff --git a/hubblestack/extmods/fdg/curl.py b/hubblestack/extmods/fdg/curl.py index 02fa4c092..bbed9b119 100644 --- a/hubblestack/extmods/fdg/curl.py +++ b/hubblestack/extmods/fdg/curl.py @@ -13,7 +13,7 @@ a system, we don't want an attacker to be able to send that data to arbitrary endpoints. """ -from __future__ import absolute_import + import logging import requests diff --git a/hubblestack/extmods/fdg/grep.py b/hubblestack/extmods/fdg/grep.py index f101cc534..651bd4d02 100644 --- a/hubblestack/extmods/fdg/grep.py +++ b/hubblestack/extmods/fdg/grep.py @@ -5,7 +5,7 @@ This fdg module allows for grepping against files and strings """ -from __future__ import absolute_import + import logging import os.path diff --git a/hubblestack/extmods/fdg/osquery.py b/hubblestack/extmods/fdg/osquery.py index 6bad1cdfe..8e3662e99 100644 --- a/hubblestack/extmods/fdg/osquery.py +++ b/hubblestack/extmods/fdg/osquery.py @@ -5,7 +5,7 @@ This fdg module allows for running osquery queries """ -from __future__ import absolute_import + import json import logging import os @@ -91,8 +91,8 @@ def _osquery(query_sql, osquery_path=None, args=None): if res['retcode'] == 0: ret = json.loads(res['stdout']) for result in ret: - for key, value in result.iteritems(): - if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'): + for key, value in result.items(): + if value and isinstance(value, str) and value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) return True, ret return False, res['stdout'] diff --git a/hubblestack/extmods/fdg/process.py b/hubblestack/extmods/fdg/process.py index 32a309000..7c6ff55fc 100644 --- a/hubblestack/extmods/fdg/process.py +++ b/hubblestack/extmods/fdg/process.py @@ -6,7 +6,7 @@ This module primarily processes and properly format the data outputted by a module to serve it to another module. """ -from __future__ import absolute_import + import logging import re @@ -72,9 +72,9 @@ def _filter_dict(dct, not equal to 2. """ ret = dct - for comp, value in filter_rules.iteritems(): + for comp, value in filter_rules.items(): try: - ret = {key: val for key, val in ret.iteritems() + ret = {key: val for key, val in ret.items() if (filter_values and _compare(comp, val, value)) or (not filter_values and _compare(comp, key, value))} except ArgumentValueError: @@ -174,7 +174,7 @@ def _filter(seq, log.error("``filter_rules`` should be of type dict") return None ret = seq - for comp, value in filter_rules.iteritems(): + for comp, value in filter_rules.items(): try: ret = [x for x in ret if _compare(comp, x, value)] except ArgumentValueError: @@ -428,7 +428,7 @@ def dict_to_list(starting_dict=None, update_chained=True, chained=None, chained_ except (AttributeError, ValueError, TypeError): log.error("Invalid arguments type.", exc_info=True) return False, None - ret = [(key, value) for key, value in chained.iteritems()] + ret = [(key, value) for key, value in chained.items()] status = bool(ret) return status, ret @@ -482,7 +482,7 @@ def _dict_convert_none(dictionary): log.error("Invalid argument type - should be dict") return None updated_dict = {} - for key, value in dictionary.iteritems(): + for key, value in dictionary.items(): if value == '': updated_dict[key] = None elif isinstance(value, dict): @@ -594,7 +594,7 @@ def _sterilize_dict(dictionary): log.error("Invalid argument type - should be dict") return None updated_dict = {} - for key, value in dictionary.iteritems(): + for key, value in dictionary.items(): if isinstance(value, dict): updated_dict[key] = _sterilize_dict(value) elif isinstance(value, (set, list, tuple)): diff --git a/hubblestack/extmods/fdg/process_status.py b/hubblestack/extmods/fdg/process_status.py index 44b83cf2e..4969a4c01 100644 --- a/hubblestack/extmods/fdg/process_status.py +++ b/hubblestack/extmods/fdg/process_status.py @@ -6,7 +6,7 @@ processes, with various options for filtering """ -from __future__ import absolute_import + import logging @@ -45,7 +45,7 @@ def _convert_to_str(process_data): ret = [] try: for process in process_data: - str_process = {str(name): str(val) for name, val in process.iteritems()} + str_process = {str(name): str(val) for name, val in process.items()} ret.append(str_process) except (TypeError, AttributeError): log.error('Invalid argument type; must be list of dicts.', exc_info=True) diff --git a/hubblestack/extmods/fdg/readfile.py b/hubblestack/extmods/fdg/readfile.py index 9f40d4cbc..8f7c1304e 100644 --- a/hubblestack/extmods/fdg/readfile.py +++ b/hubblestack/extmods/fdg/readfile.py @@ -5,7 +5,7 @@ This fdg module allows for reading in the contents of files, with various options for format and filtering. """ -from __future__ import absolute_import + import json as _json import logging diff --git a/hubblestack/extmods/fdg/time_sync.py b/hubblestack/extmods/fdg/time_sync.py index dd6c866f6..6c8b25340 100644 --- a/hubblestack/extmods/fdg/time_sync.py +++ b/hubblestack/extmods/fdg/time_sync.py @@ -5,7 +5,7 @@ NTP servers for differences bigger than 15 minutes. ''' -from __future__ import absolute_import + import logging import salt.utils.platform diff --git a/hubblestack/extmods/fileserver/azurefs.py b/hubblestack/extmods/fileserver/azurefs.py index 007da53b5..cbc95603c 100644 --- a/hubblestack/extmods/fileserver/azurefs.py +++ b/hubblestack/extmods/fileserver/azurefs.py @@ -47,7 +47,7 @@ """ # Import python libs -from __future__ import absolute_import + from distutils.version import LooseVersion import base64 import json diff --git a/hubblestack/extmods/fileserver/s3fs.py b/hubblestack/extmods/fileserver/s3fs.py index 2cddd1dde..bc3519a3a 100644 --- a/hubblestack/extmods/fileserver/s3fs.py +++ b/hubblestack/extmods/fileserver/s3fs.py @@ -79,7 +79,7 @@ """ # Import python libs -from __future__ import absolute_import, print_function, unicode_literals + import datetime import os import time @@ -683,7 +683,7 @@ def _get_file(): path_style=s3_key_kwargs['path_style'], https_enable=s3_key_kwargs['https_enable']) if ret: - for header_name, header_value in ret['headers'].items(): + for header_name, header_value in list(ret['headers'].items()): header_name = header_name.strip() header_value = header_value.strip() if six.text_type(header_name).lower() == 'last-modified': diff --git a/hubblestack/extmods/grains/cloud_details.py b/hubblestack/extmods/grains/cloud_details.py index dbf4ff4bc..9982e543b 100644 --- a/hubblestack/extmods/grains/cloud_details.py +++ b/hubblestack/extmods/grains/cloud_details.py @@ -70,7 +70,7 @@ def _get_aws_details(): r = requests.get('http://169.254.169.254/latest/meta-data/local-hostname', headers=aws_token_header, timeout=3, proxies=proxies) if r.status_code == requests.codes.ok: aws_extra['cloud_private_hostname'] = r.text - for key in aws_extra.keys(): + for key in list(aws_extra.keys()): if not aws_extra[key]: aws_extra.pop(key) diff --git a/hubblestack/extmods/grains/configgrains.py b/hubblestack/extmods/grains/configgrains.py index da1a7e9ae..76466bc88 100644 --- a/hubblestack/extmods/grains/configgrains.py +++ b/hubblestack/extmods/grains/configgrains.py @@ -43,7 +43,7 @@ def configgrains(): grains_to_make = __salt__['config.get']('config_to_grains', default=[]) for grain in grains_to_make: - for grain_key, grain_value in grain.iteritems(): + for grain_key, grain_value in grain.items(): grain_value = __salt__['config.get'](grain_value, default=None) if grain_value: grains[grain_key] = grain_value diff --git a/hubblestack/extmods/grains/default_gw.py b/hubblestack/extmods/grains/default_gw.py index ae2381078..8898d70f4 100644 --- a/hubblestack/extmods/grains/default_gw.py +++ b/hubblestack/extmods/grains/default_gw.py @@ -15,7 +15,7 @@ ip6_gw: True # True/False if default ipv6 gateway ip_gw: True # True if either of the above is True, False otherwise """ -from __future__ import absolute_import + import logging diff --git a/hubblestack/extmods/grains/fqdn.py b/hubblestack/extmods/grains/fqdn.py index 3c4a4eef2..3c7a9d47f 100644 --- a/hubblestack/extmods/grains/fqdn.py +++ b/hubblestack/extmods/grains/fqdn.py @@ -55,7 +55,7 @@ def _find_addr(interfaces): # Fallback to "best guess" filtered_interfaces = {} # filter out empty, lo, and docker0 - for interface, ips in interfaces.iteritems(): + for interface, ips in interfaces.items(): if not ips or interface in ('lo', 'docker0'): continue filtered_interfaces[interface] = ips @@ -65,16 +65,15 @@ def _find_addr(interfaces): if ip_addr != '127.0.0.1': return {'local_ip4': ip_addr} # Use .*0 if present - for interface, ips in filtered_interfaces.iteritems(): + for interface, ips in filtered_interfaces.items(): if '0' in interface: for ip_addr in ips: if ip_addr != '127.0.0.1': return {'local_ip4': ip_addr} # Use whatever isn't 127.0.0.1 - for interface, ips in filtered_interfaces.iteritems(): + for interface, ips in filtered_interfaces.items(): for ip_addr in ips: if ip_addr != '127.0.0.1': return {'local_ip4': ip_addr} - # Give up return {'local_ip4', ''} diff --git a/hubblestack/extmods/modules/audit.py b/hubblestack/extmods/modules/audit.py index 749401d03..8de7459aa 100644 --- a/hubblestack/extmods/modules/audit.py +++ b/hubblestack/extmods/modules/audit.py @@ -94,7 +94,7 @@ it was a success or a failure, and ``data_dict`` is a dictionary of any information that should be added to the check's data dictionary in the return. """ -from __future__ import absolute_import + import fnmatch import logging @@ -189,7 +189,7 @@ def audit(audit_files=None, succinct_ret = {'Success': [], 'Failure': [], 'Skipped': []} - for success_type, checks in ret.iteritems(): + for success_type, checks in ret.items(): for check in checks: succinct_ret[success_type].append( {check['tag']: check.get('description', '')}) @@ -285,7 +285,7 @@ def _get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) @@ -355,10 +355,10 @@ def _run_audit(ret, audit_data, tags, labels, audit_file): :return: Returns the updated ``ret`` object """ - for audit_id, data in audit_data.iteritems(): + for audit_id, data in audit_data.items(): log.debug('Executing audit id %s in audit file %s', audit_id, audit_file) try: - module = data.keys()[0] + module = list(data.keys())[0] data = data[module] if not isinstance(data, dict): log.error('Audit data with id %s from file %s not formatted correctly', diff --git a/hubblestack/extmods/modules/conf_publisher.py b/hubblestack/extmods/modules/conf_publisher.py index 4756046b8..74c120049 100644 --- a/hubblestack/extmods/modules/conf_publisher.py +++ b/hubblestack/extmods/modules/conf_publisher.py @@ -63,7 +63,7 @@ def _filter_config(opts_to_log, remove_dots=True): patterns_to_filter = ["password", "token", "passphrase", "privkey", "keyid", "s3.key"] filtered_conf = _remove_sensitive_info(opts_to_log, patterns_to_filter) if remove_dots: - for key in filtered_conf.keys(): + for key in list(filtered_conf.keys()): if '.' in key: filtered_conf[key.replace('.', '_')] = filtered_conf.pop(key) return filtered_conf @@ -76,7 +76,7 @@ def _remove_sensitive_info(obj, patterns_to_filter): if isinstance(obj, dict): obj = { key: _remove_sensitive_info(value, patterns_to_filter) - for key, value in obj.iteritems() + for key, value in obj.items() if not any(patt in key for patt in patterns_to_filter)} elif isinstance(obj, list): obj = [_remove_sensitive_info(item, patterns_to_filter) for item in obj] diff --git a/hubblestack/extmods/modules/fdg.py b/hubblestack/extmods/modules/fdg.py index ef5f0df16..6714fcf4e 100644 --- a/hubblestack/extmods/modules/fdg.py +++ b/hubblestack/extmods/modules/fdg.py @@ -114,7 +114,7 @@ will end and any ``return`` keywords will be evaluated as we move back up the call chain. """ -from __future__ import absolute_import + import logging import os import yaml @@ -214,7 +214,7 @@ def top(fdg_topfile='salt://fdg/top.fdg'): ret = {} for fdg_file in fdg_routines: if isinstance(fdg_file, dict): - for key, val in fdg_file.iteritems(): + for key, val in fdg_file.items(): retkey, retval = fdg(_fdg_saltify(key), val) ret[retkey] = retval else: @@ -381,7 +381,7 @@ def _get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) diff --git a/hubblestack/extmods/modules/hstatus.py b/hubblestack/extmods/modules/hstatus.py index 9446bad44..b5a2d2703 100644 --- a/hubblestack/extmods/modules/hstatus.py +++ b/hubblestack/extmods/modules/hstatus.py @@ -40,7 +40,7 @@ def msg_counts(pat=MSG_COUNTS_PAT, emit_self=False, sourcetype=SOURCETYPE): pat = re.compile(pat) ret = list() # events to return for bucket_set in hubblestack.status.HubbleStatus.short('all'): - for key, val in bucket_set.iteritems(): + for key, val in bucket_set.items(): try: # should be at least one count if val['count'] < 1: diff --git a/hubblestack/extmods/modules/hubble.py b/hubblestack/extmods/modules/hubble.py index 0117a8c4d..6146747eb 100644 --- a/hubblestack/extmods/modules/hubble.py +++ b/hubblestack/extmods/modules/hubble.py @@ -11,7 +11,7 @@ - hubblestack:nova:autoload - hubblestack:nova:autosync """ -from __future__ import absolute_import + import logging import os @@ -298,7 +298,7 @@ def _run_audit(configs, tags, debug, labels, **kwargs): # have available with the data list, so data will be processed multiple # times. However, for the scale we're working at this should be fine. # We can revisit if this ever becomes a big bottleneck - for key, func in __nova__._dict.iteritems(): + for key, func in __nova__._dict.items(): try: ret = func(data_list, tags, labels, **kwargs) except Exception: @@ -318,7 +318,7 @@ def _run_audit(configs, tags, debug, labels, **kwargs): continue # Merge in the results - for ret_key, ret_val in ret.iteritems(): + for ret_key, ret_val in ret.items(): if ret_key not in results: results[ret_key] = [] results[ret_key].extend(ret_val) @@ -333,7 +333,7 @@ def _run_audit(configs, tags, debug, labels, **kwargs): for failure_index in reversed(sorted(set(failures_to_remove))): results['Failure'].pop(failure_index) - for key in results.keys(): + for key in list(results.keys()): if not results[key]: results.pop(key) @@ -381,7 +381,7 @@ def _build_processed_controls(data_list, debug): if isinstance(control, str): processed_controls[control] = {} else: # dict - for control_tag, control_data in control.iteritems(): + for control_tag, control_data in control.items(): if isinstance(control_data, str): processed_controls[control_tag] = {'reason': control_data} else: # dict @@ -517,7 +517,7 @@ def top(topfile='top.nova', return results # Run the audits - for tag, data in data_by_tag.iteritems(): + for tag, data in data_by_tag.items(): ret = audit(configs=data, tags=tag, verbose=verbose, @@ -527,7 +527,7 @@ def top(topfile='top.nova', labels=labels) # Merge in the results - for key, val in ret.iteritems(): + for key, val in ret.items(): if key not in results: results[key] = [] results[key].extend(val) @@ -553,12 +553,12 @@ def _build_data_by_tag(topfile, results): top_data = _get_top_data(topfile) for data in top_data: - if isinstance(data, basestring): + if isinstance(data, str): if '*' not in data_by_tag: data_by_tag['*'] = [] data_by_tag['*'].append(data) elif isinstance(data, dict): - for key, tag in data.iteritems(): + for key, tag in data.items(): if tag not in data_by_tag: data_by_tag[tag] = [] data_by_tag[tag].append(key) @@ -581,7 +581,7 @@ def _clean_up_results(results, show_success): if show_success was not passed, adding an error message if results is empty """ - for key in results.keys(): + for key in list(results.keys()): if not results[key]: results.pop(key) @@ -675,9 +675,9 @@ def load(): global __nova__ __nova__ = NovaLazyLoader(_hubble_dir(), __opts__, __grains__, __pillar__, __salt__) - ret = {'loaded': __nova__._dict.keys(), + ret = {'loaded': list(__nova__._dict.keys()), 'missing': __nova__.missing_modules, - 'data': __nova__.__data__.keys(), + 'data': list(__nova__.__data__.keys()), 'missing_data': __nova__.__missing_data__} return ret @@ -749,7 +749,7 @@ def _get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 8779192bc..ed4c0985d 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -21,7 +21,7 @@ - query_name: rpm_packages query: select rpm.*, t.iso_8601 from rpm_packages as rpm join time as t; """ -from __future__ import absolute_import + import collections import copy @@ -171,7 +171,7 @@ def _build_baseline_osquery_data(report_version_with_day): ret.append( {'fallback_pkgs': { 'data': [{'name': k, 'version': v} - for k, v in __salt__['pkg.list_pkgs']().iteritems()], + for k, v in __salt__['pkg.list_pkgs']().items()], 'result': True}}) uptime = __salt__['status.uptime']() if isinstance(uptime, dict): @@ -224,7 +224,7 @@ def _run_osquery_queries(query_data, verbose): ret = [] timing = {} success = True - for name, query in query_data.iteritems(): + for name, query in query_data.items(): query['query_name'] = name query_sql = query.get('query') if not query_sql: @@ -254,12 +254,12 @@ def _update_osquery_results(ret): Returns the updated version. """ for data in ret: - for _query_name, query_ret in data.iteritems(): + for _query_name, query_ret in data.items(): if 'data' not in query_ret: continue for result in query_ret['data']: - for key, value in result.iteritems(): - if value and isinstance(value, basestring) and\ + for key, value in result.items(): + if value and isinstance(value, str) and\ value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) @@ -472,13 +472,13 @@ def _update_event_data(ret): obj = json.loads(event_data) if 'action' in obj and obj['action'] == 'snapshot': for result in obj['snapshot']: - for key, value in result.iteritems(): - if value and isinstance(value, basestring) and \ + for key, value in result.items(): + if value and isinstance(value, str) and \ value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) elif 'action' in obj: - for key, value in obj['columns'].iteritems(): - if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'): + for key, value in obj['columns'].items(): + if value and isinstance(value, str) and value.startswith('__JSONIFY__'): obj['columns'][key] = json.loads(value[len('__JSONIFY__'):]) n_ret.append(obj) @@ -621,7 +621,7 @@ def _get_top_data(topfile): ret = [] for topmatch in topdata: - for match, data in topmatch.iteritems(): + for match, data in topmatch.items(): if __salt__['match.compound'](match): ret.extend(data) @@ -875,7 +875,7 @@ def _mask_object_helper(object_to_be_masked, perform_masking_kwargs, column, que _mask_interactive_shell_data(data, kwargs) else: kwargs['custom_args']['log_error'] = False - for query_name, query_ret in obj.iteritems(): + for query_name, query_ret in obj.items(): data = query_ret['data'] _mask_interactive_shell_data(data, kwargs) @@ -995,14 +995,14 @@ def _mask_event_data_helper(event_data, query_name, column, perform_masking_kwar if mask_column and isinstance(mask_column, list): blacklisted_object = _custom_blacklisted_object(blacklisted_object, mask_column) if column not in event_data or \ - (isinstance(event_data[column], basestring) and + (isinstance(event_data[column], str) and event_data[column].strip() != ''): if custom_args['log_error']: log.error('masking data references a missing column %s in query %s', column, query_name) if custom_args['should_break']: return False, blacklisted_object, event_data - if isinstance(event_data[column], basestring): + if isinstance(event_data[column], str): # If column is of 'string' type, then replace pattern in-place # No need for recursion here value = event_data[column] diff --git a/hubblestack/extmods/modules/nova_loader.py b/hubblestack/extmods/modules/nova_loader.py index a28682b63..753e74833 100644 --- a/hubblestack/extmods/modules/nova_loader.py +++ b/hubblestack/extmods/modules/nova_loader.py @@ -6,7 +6,7 @@ """ # Import python libs -from __future__ import absolute_import + import os import imp import sys @@ -39,6 +39,7 @@ # Import 3rd-party libs import salt.ext.six as six +import importlib try: import pkg_resources HAS_PKG_RESOURCES = True @@ -114,7 +115,7 @@ class LazyDict(collections.MutableMapping): def __init__(self): self.clear() - def __nonzero__(self): + def __bool__(self): # we are zero if dict is empty and loaded is true return bool(self._dict or not self.loaded) @@ -1395,7 +1396,7 @@ def _reload_submodules(self, mod): for submodule in submodules: # it is a submodule if the name is in a namespace under mod if submodule.__name__.startswith(mod.__name__ + '.'): - reload(submodule) + importlib.reload(submodule) self._reload_submodules(submodule) def _load_module(self, name): diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index 6efe6372e..19af207a8 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -12,7 +12,7 @@ """ # Import Python libs -from __future__ import absolute_import + import types import base64 import collections @@ -558,8 +558,8 @@ def _wrapped(val): the_list = [] for e in excludes: if isinstance(e,dict): - if e.values()[0].get('regex'): - r = e.keys()[0] + if list(e.values())[0].get('regex'): + r = list(e.keys())[0] try: c = re.compile(r) the_list.append(re_wrapper(c)) @@ -567,7 +567,7 @@ def _wrapped(val): log.warn('Failed to compile regex "%s": %s', r,e) continue else: - e = e.keys()[0] + e = list(e.keys())[0] if '*' in e: the_list.append(fn_wrapper(e)) else: @@ -972,7 +972,7 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): return dest else: try: - for k in upd.keys(): + for k in list(upd.keys()): dest[k] = upd[k] except AttributeError: # this mapping is not a dict @@ -1036,7 +1036,7 @@ def get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) diff --git a/hubblestack/extmods/modules/safecommand.py b/hubblestack/extmods/modules/safecommand.py index cea31251c..e14703096 100644 --- a/hubblestack/extmods/modules/safecommand.py +++ b/hubblestack/extmods/modules/safecommand.py @@ -14,7 +14,7 @@ This module allows for this functionality. """ -from __future__ import absolute_import + import logging from salt.exceptions import CommandExecutionError diff --git a/hubblestack/extmods/modules/sysexit.py b/hubblestack/extmods/modules/sysexit.py index 4cc348890..d85b583eb 100644 --- a/hubblestack/extmods/modules/sysexit.py +++ b/hubblestack/extmods/modules/sysexit.py @@ -2,7 +2,7 @@ """ intended for testing, this module's sole purpose is to cause the running daemon to exit gracefully within a scheduled timeframe """ -from __future__ import absolute_import + import logging import sys diff --git a/hubblestack/extmods/modules/win_pulsar.py b/hubblestack/extmods/modules/win_pulsar.py index 7a820f159..91e0164d1 100644 --- a/hubblestack/extmods/modules/win_pulsar.py +++ b/hubblestack/extmods/modules/win_pulsar.py @@ -4,7 +4,7 @@ It will then scan the ntfs journal for changes to those folders and report when it finds one. """ -from __future__ import absolute_import + from time import mktime, strptime, time import collections @@ -535,7 +535,7 @@ def get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) diff --git a/hubblestack/extmods/modules/win_pulsar_winaudit.py b/hubblestack/extmods/modules/win_pulsar_winaudit.py index 193b59949..2d6046e3e 100644 --- a/hubblestack/extmods/modules/win_pulsar_winaudit.py +++ b/hubblestack/extmods/modules/win_pulsar_winaudit.py @@ -4,7 +4,7 @@ It will then scan the event log for changes to those folders and report when it finds one. """ -from __future__ import absolute_import + import collections import fnmatch @@ -255,8 +255,8 @@ def _should_append(config, path, event): _append = True if isinstance(config[path], dict) and 'exclude' in config[path]: for exclude in config[path]['exclude']: - if isinstance(exclude, dict) and exclude.values()[0].get('regex', False): - if re.search(exclude.keys()[0], event['Object Name']): + if isinstance(exclude, dict) and list(exclude.values())[0].get('regex', False): + if re.search(list(exclude.keys())[0], event['Object Name']): _append = False else: if fnmatch.fnmatch(event['Object Name'], exclude): @@ -669,7 +669,7 @@ def get_top_data(topfile): ret = [] - for match, data in topdata.iteritems(): + for match, data in topdata.items(): if __salt__['match.compound'](match): ret.extend(data) diff --git a/hubblestack/extmods/returners/graylog_nebula_return.py b/hubblestack/extmods/returners/graylog_nebula_return.py index 30158aa24..acb533fb1 100644 --- a/hubblestack/extmods/returners/graylog_nebula_return.py +++ b/hubblestack/extmods/returners/graylog_nebula_return.py @@ -21,7 +21,7 @@ """ -from __future__ import absolute_import + import json import requests @@ -53,7 +53,7 @@ def returner(ret): for opts in opts_list: for query in ret['return']: - for query_name, value in query.items(): + for query_name, value in list(query.items()): for query_data in value['data']: args = {'query': query_name, 'job_id': ret['jid'], diff --git a/hubblestack/extmods/returners/graylog_nova_return.py b/hubblestack/extmods/returners/graylog_nova_return.py index 577bac6da..cc3057754 100644 --- a/hubblestack/extmods/returners/graylog_nova_return.py +++ b/hubblestack/extmods/returners/graylog_nova_return.py @@ -118,7 +118,7 @@ def _generate_event(args, cloud_details, custom_fields, compliance=False, data=N if not isinstance(data[args['check_id']], dict): event.update({'description': data[args['check_id']]}) elif 'description' in data[args['check_id']]: - for key, value in data[args['check_id']].iteritems(): + for key, value in data[args['check_id']].items(): if key not in ['tag']: event[key] = value else: @@ -182,7 +182,7 @@ def _publish_data(args, checks, check_result, cloud_details, opts): graylog server """ for data in checks: - check_id = data.keys()[0] + check_id = list(data.keys())[0] args['check_result'] = check_result args['check_id'] = check_id event = _generate_event(data=data, args=args, cloud_details=cloud_details, diff --git a/hubblestack/extmods/returners/logstash_nebula_return.py b/hubblestack/extmods/returners/logstash_nebula_return.py index 48fb4ba52..4c5861bbd 100644 --- a/hubblestack/extmods/returners/logstash_nebula_return.py +++ b/hubblestack/extmods/returners/logstash_nebula_return.py @@ -54,7 +54,7 @@ def returner(ret): break for query in ret['return']: - for query_name, query_results in query.iteritems(): + for query_name, query_results in query.items(): for query_result in query_results['data']: args = {'query': query_name, 'job_id': ret['jid'], diff --git a/hubblestack/extmods/returners/logstash_nova_return.py b/hubblestack/extmods/returners/logstash_nova_return.py index 0b4acd471..8a1d8f3d1 100644 --- a/hubblestack/extmods/returners/logstash_nova_return.py +++ b/hubblestack/extmods/returners/logstash_nova_return.py @@ -143,7 +143,7 @@ def _generate_event(args, cloud_details, custom_fields, compliance=False, data=N if not isinstance(data[args['check_id']], dict): event.update({'description': data[args['check_id']]}) elif 'description' in data[args['check_id']]: - for key, value in data[args['check_id']].iteritems(): + for key, value in data[args['check_id']].items(): if key not in ['tag']: event[key] = value else: @@ -170,7 +170,7 @@ def _publish_data(args, checks, check_result, cloud_details, opts): Helper function that goes over the failure/success checks and publishes the event to logstash """ for data in checks: - check_id = data.keys()[0] + check_id = list(data.keys())[0] args['check_result'] = check_result args['check_id'] = check_id event = _generate_event(custom_fields=opts['custom_fields'], data=data, args=args, diff --git a/hubblestack/extmods/returners/slack_pulsar_returner.py b/hubblestack/extmods/returners/slack_pulsar_returner.py index d2910675a..957b869fd 100644 --- a/hubblestack/extmods/returners/slack_pulsar_returner.py +++ b/hubblestack/extmods/returners/slack_pulsar_returner.py @@ -54,12 +54,12 @@ channel: RoomName """ -from __future__ import absolute_import + # Import Python libs import pprint import logging -import urllib +import urllib.request, urllib.parse, urllib.error # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urljoin as _urljoin @@ -241,7 +241,7 @@ def _post_message(channel, api_key=api_key, method='POST', header_dict={'Content-Type': 'application/x-www-form-urlencoded'}, - data=urllib.urlencode(parameters)) + data=urllib.parse.urlencode(parameters)) log.debug('result %s', result) diff --git a/hubblestack/extmods/returners/splunk_fdg_return.py b/hubblestack/extmods/returners/splunk_fdg_return.py index a7e094b29..17e55d19c 100644 --- a/hubblestack/extmods/returners/splunk_fdg_return.py +++ b/hubblestack/extmods/returners/splunk_fdg_return.py @@ -92,7 +92,7 @@ def returner(ret): args, kwargs = make_hec_args(opts) hec = http_event_collector(*args, **kwargs) - for fdg_info, fdg_results in data.iteritems(): + for fdg_info, fdg_results in data.items(): if not isinstance(fdg_results, list): fdg_results = [fdg_results] @@ -132,7 +132,7 @@ def _generate_event(fdg_args, args, starting_chained, cloud_details, custom_fiel custom_field_value = __salt__['config.get'](custom_field, '') if isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) return event diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index 4babf5981..8e33e465c 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -84,7 +84,7 @@ def returner(ret): hec = http_event_collector(*args, **kwargs) for query in ret['return']: - for query_name, query_results in query.iteritems(): + for query_name, query_results in query.items(): if 'data' not in query_results: query_results['data'] = [{'error': 'result missing'}] for query_result in query_results['data']: @@ -158,7 +158,7 @@ def _generate_event(host_args, query_result, query_name, custom_fields, cloud_de for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) elif isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index 8b8bf0448..48cf2c7aa 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -166,7 +166,7 @@ def _generate_event(args, cloud_details, custom_fields, check_type=None, data=No if not isinstance(data[args['check_id']], dict): event.update({'description': data[args['check_id']]}) elif 'description' in data[args['check_id']]: - for key, value in data[args['check_id']].iteritems(): + for key, value in data[args['check_id']].items(): if key not in ['tag']: event[key] = value event.update({'minion_id': args['minion_id'], @@ -181,7 +181,7 @@ def _generate_event(args, cloud_details, custom_fields, check_type=None, data=No custom_field_value = __salt__['config.get'](custom_field, '') if isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) if check_type == 'Success': @@ -226,7 +226,7 @@ def _publish_data(args, checks, check_result, cloud_details, opts): Helper function that goes over the failure/success checks and publishes the event to Splunk """ for data in checks: - check_id = data.keys()[0] + check_id = list(data.keys())[0] args['check_result'] = check_result args['check_id'] = check_id event = _generate_event(data=data, args=args, cloud_details=cloud_details, diff --git a/hubblestack/extmods/returners/splunk_osqueryd_return.py b/hubblestack/extmods/returners/splunk_osqueryd_return.py index 6fb6d57a2..65115f1d6 100644 --- a/hubblestack/extmods/returners/splunk_osqueryd_return.py +++ b/hubblestack/extmods/returners/splunk_osqueryd_return.py @@ -208,7 +208,7 @@ def _update_event(custom_fields, event): custom_field_value = __salt__['config.get'](custom_field, '') if isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) # Remove any empty fields from the event payload diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index e8a59e0ed..667539714 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -312,7 +312,7 @@ def _update_event(custom_fields, host_args, cloud_details, event): custom_field_value = __salt__['config.get'](custom_field, '') if isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) # Remove any empty fields from the event payload remove_keys = [k for k in event if event[k] == ""] diff --git a/hubblestack/extmods/returners/sqlite.py b/hubblestack/extmods/returners/sqlite.py index bbe272111..a7b125dbe 100644 --- a/hubblestack/extmods/returners/sqlite.py +++ b/hubblestack/extmods/returners/sqlite.py @@ -2,7 +2,7 @@ """ Return hubble data to sqlite (intended for testing) """ -from __future__ import absolute_import + import json import logging diff --git a/hubblestack/extmods/returners/sumo_nebula_return.py b/hubblestack/extmods/returners/sumo_nebula_return.py index 3937a31e9..b429e4bf3 100644 --- a/hubblestack/extmods/returners/sumo_nebula_return.py +++ b/hubblestack/extmods/returners/sumo_nebula_return.py @@ -41,7 +41,7 @@ def returner(ret): for opts in opts_list: sumo_nebula_return = opts['sumo_nebula_return'] for query in data: - for query_name, query_results in query.iteritems(): + for query_name, query_results in query.items(): if 'data' not in query_results: query_results['data'] = [{'error': 'result missing'}] for query_result in query_results['data']: diff --git a/hubblestack/extmods/returners/sumo_nova_return.py b/hubblestack/extmods/returners/sumo_nova_return.py index 5622c0d44..be4524aae 100644 --- a/hubblestack/extmods/returners/sumo_nova_return.py +++ b/hubblestack/extmods/returners/sumo_nova_return.py @@ -98,7 +98,7 @@ def _generate_event(args, cloud_details, compliance=False, data=None): if not isinstance(data[args['check_id']], dict): event.update({'description': data[args['check_id']]}) elif 'description' in data[args['check_id']]: - for key, value in data[args['check_id']].iteritems(): + for key, value in data[args['check_id']].items(): if key not in ['tag']: event[key] = value event.update({'minion_id': args['minion_id'], @@ -122,7 +122,7 @@ def _publish_data(args, checks, check_result, cloud_details, sumo_nova_return): Helper function that goes over the failure/success checks and publishes the event to sumo """ for data in checks: - check_id = data.keys()[0] + check_id = list(data.keys())[0] args['check_result'] = check_result args['check_id'] = check_id event = _generate_event(data=data, args=args, cloud_details=cloud_details) diff --git a/hubblestack/extmods/utils/s3.py b/hubblestack/extmods/utils/s3.py index f170e5635..8f34a321d 100644 --- a/hubblestack/extmods/utils/s3.py +++ b/hubblestack/extmods/utils/s3.py @@ -4,7 +4,7 @@ :depends: requests """ -from __future__ import absolute_import, print_function, unicode_literals + # Import Python libs import logging diff --git a/hubblestack/files/hubblestack_nova/command.py b/hubblestack/files/hubblestack_nova/command.py index 51bb0d376..331129a9d 100644 --- a/hubblestack/files/hubblestack_nova/command.py +++ b/hubblestack/files/hubblestack_nova/command.py @@ -63,7 +63,7 @@ # Description will be output with the results description: '/home should be nodev' """ -from __future__ import absolute_import + import logging import fnmatch @@ -115,7 +115,7 @@ def audit(data_list, tags, labels, **kwargs): continue command_results = [] for command_data in tag_data['commands']: - for command, command_args in command_data.iteritems(): + for command, command_args in command_data.items(): if 'shell' in command_args: cmd_ret = __salt__['cmd.run'](command, python_shell=True, @@ -173,7 +173,7 @@ def _merge_yaml(ret, data, profile=None): if 'command' not in ret: ret['command'] = [] if 'command' in data: - for key, val in data['command'].iteritems(): + for key, val in data['command'].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['command'].append({key: val}) @@ -188,7 +188,7 @@ def _get_tags(data): distro = __grains__.get('osfinger') for audit_dict in data.get('command', []): # command:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # command:0:nodev tags_dict = audit_data.get('data', {}) # command:0:nodev:data diff --git a/hubblestack/files/hubblestack_nova/fdg.py b/hubblestack/files/hubblestack_nova/fdg.py index 7c75e1511..249d2ee6d 100644 --- a/hubblestack/files/hubblestack_nova/fdg.py +++ b/hubblestack/files/hubblestack_nova/fdg.py @@ -40,7 +40,7 @@ fdg run will be considered. Regardless, the ``true_for_success`` argument will be respected. """ -from __future__ import absolute_import + import logging import fnmatch @@ -121,7 +121,7 @@ def _merge_yaml(ret, data, profile=None): """ if 'fdg' not in ret: ret['fdg'] = [] - for key, val in data.get('fdg', {}).iteritems(): + for key, val in data.get('fdg', {}).items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['fdg'].append({key: val}) @@ -136,7 +136,7 @@ def _get_tags(data): distro = __grains__.get('osfinger') for audit_dict in data.get('fdg', {}): # fdg:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # fdg:0:id tags_dict = audit_data.get('data', {}) # fdg:0:id:data diff --git a/hubblestack/files/hubblestack_nova/firewall.py b/hubblestack/files/hubblestack_nova/firewall.py index abd859673..c86ebada1 100644 --- a/hubblestack/files/hubblestack_nova/firewall.py +++ b/hubblestack/files/hubblestack_nova/firewall.py @@ -75,7 +75,7 @@ (https://github.com/saltstack/salt/blob/develop/salt/modules/iptables.py) """ -from __future__ import absolute_import + import logging import fnmatch @@ -193,7 +193,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('firewall', {}): if topkey not in ret['firewall']: ret['firewall'][topkey] = [] - for key, val in data['firewall'][topkey].iteritems(): + for key, val in data['firewall'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['firewall'][topkey].append({key: val}) @@ -202,9 +202,9 @@ def _merge_yaml(ret, data, profile=None): def _get_tags(data): ret = {} - for toplist, toplevel in data.get('firewall', {}).iteritems(): + for toplist, toplevel in data.get('firewall', {}).items(): for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): tags_dict = audit_data.get('data', {}) tag = tags_dict.pop('tag') if tag not in ret: diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py index 0317a0698..f36362894 100644 --- a/hubblestack/files/hubblestack_nova/grep.py +++ b/hubblestack/files/hubblestack_nova/grep.py @@ -51,7 +51,7 @@ will be considered a non-match (success for blacklist, failure for whitelist). If the file exists, this setting is ignored. """ -from __future__ import absolute_import + import logging import fnmatch @@ -210,7 +210,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('grep', {}): if topkey not in ret['grep']: ret['grep'][topkey] = [] - for key, val in data['grep'][topkey].iteritems(): + for key, val in data['grep'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['grep'][topkey].append({key: val}) @@ -223,11 +223,11 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get('grep', {}).iteritems(): + for toplist, toplevel in data.get('grep', {}).items(): # grep:blacklist for audit_dict in toplevel: # grep:blacklist:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # grep:blacklist:0:telnet tags_dict = audit_data.get('data', {}) # grep:blacklist:0:telnet:data @@ -249,11 +249,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index 6774a4808..b491d4f26 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -38,7 +38,7 @@ # Description will be output with the results description: '/home should be nodev' """ -from __future__ import absolute_import + import logging import fnmatch @@ -136,7 +136,7 @@ def _merge_yaml(ret, data, profile=None): if 'misc' not in ret: ret['misc'] = [] if 'misc' in data: - for key, val in data['misc'].iteritems(): + for key, val in data['misc'].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['misc'].append({key: val}) @@ -151,7 +151,7 @@ def _get_tags(data): distro = __grains__.get('osfinger') for audit_dict in data.get('misc', []): # misc:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # misc:0:nodev tags_dict = audit_data.get('data', {}) # misc:0:nodev:data @@ -449,7 +449,7 @@ def check_duplicate_uids(reason=''): """ uids = _execute_shell_command("cat /etc/passwd | cut -f3 -d\":\"", python_shell=True).strip() uids = uids.split('\n') if uids != "" else [] - duplicate_uids = [k for k, v in Counter(uids).items() if v > 1] + duplicate_uids = [k for k, v in list(Counter(uids).items()) if v > 1] if duplicate_uids is None or duplicate_uids == []: return True return str(duplicate_uids) @@ -461,7 +461,7 @@ def check_duplicate_gids(reason=''): """ gids = _execute_shell_command("cat /etc/group | cut -f3 -d\":\"", python_shell=True).strip() gids = gids.split('\n') if gids != "" else [] - duplicate_gids = [k for k, v in Counter(gids).items() if v > 1] + duplicate_gids = [k for k, v in list(Counter(gids).items()) if v > 1] if duplicate_gids is None or duplicate_gids == []: return True return str(duplicate_gids) @@ -473,7 +473,7 @@ def check_duplicate_unames(reason=''): """ unames = _execute_shell_command("cat /etc/passwd | cut -f1 -d\":\"", python_shell=True).strip() unames = unames.split('\n') if unames != "" else [] - duplicate_unames = [k for k, v in Counter(unames).items() if v > 1] + duplicate_unames = [k for k, v in list(Counter(unames).items()) if v > 1] if duplicate_unames is None or duplicate_unames == []: return True return str(duplicate_unames) @@ -485,7 +485,7 @@ def check_duplicate_gnames(reason=''): """ gnames = _execute_shell_command("cat /etc/group | cut -f1 -d\":\"", python_shell=True).strip() gnames = gnames.split('\n') if gnames != "" else [] - duplicate_gnames = [k for k, v in Counter(gnames).items() if v > 1] + duplicate_gnames = [k for k, v in list(Counter(gnames).items()) if v > 1] if duplicate_gnames is None or duplicate_gnames == []: return True return str(duplicate_gnames) @@ -963,7 +963,7 @@ def mail_conf_check(reason=''): mail_addresses = _execute_shell_command("grep '^[[:blank:]]*inet_interfaces' /etc/postfix/main.cf | awk -F'=' '{print $2}'", python_shell=True).strip() mail_addresses = str(mail_addresses) mail_addresses = mail_addresses.split(',') if mail_addresses != "" else [] - mail_addresses = map(str.strip, mail_addresses) + mail_addresses = list(map(str.strip, mail_addresses)) invalid_addresses = list(set(mail_addresses) - set(valid_addresses)) return str(invalid_addresses) if invalid_addresses != [] else True diff --git a/hubblestack/files/hubblestack_nova/mount.py b/hubblestack/files/hubblestack_nova/mount.py index c49e5aff7..7275346cc 100644 --- a/hubblestack/files/hubblestack_nova/mount.py +++ b/hubblestack/files/hubblestack_nova/mount.py @@ -27,7 +27,7 @@ labels: - critical """ -from __future__ import absolute_import + import logging import fnmatch @@ -145,7 +145,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('mount', {}): if topkey not in ret['mount']: ret['mount'][topkey] = [] - for key, val in data['mount'][topkey].iteritems(): + for key, val in data['mount'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['mount'][topkey].append({key: val}) @@ -160,11 +160,11 @@ def _get_tags(data): ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get('mount', {}).iteritems(): + for toplist, toplevel in data.get('mount', {}).items(): # mount:blacklist for audit_dict in toplevel: # mount:blacklist:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # mount:blacklist:0:telnet tags_dict = audit_data.get('data', {}) # mount:blacklist:0:telnet:data @@ -186,11 +186,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/netstat.py b/hubblestack/files/hubblestack_nova/netstat.py index 8390f7566..2618fdf9b 100644 --- a/hubblestack/files/hubblestack_nova/netstat.py +++ b/hubblestack/files/hubblestack_nova/netstat.py @@ -14,7 +14,7 @@ - 127.0.0.1:80 - 0.0.0.0:80 """ -from __future__ import absolute_import + import copy import fnmatch @@ -40,7 +40,7 @@ def audit(data_list, tags, labels, debug=True, **kwargs): __tags__ = {} for profile, data in data_list: if 'netstat' in data: - for check, check_args in data['netstat'].iteritems(): + for check, check_args in data['netstat'].items(): if 'address' in check_args: tag_args = copy.deepcopy(check_args) tag_args['id'] = check diff --git a/hubblestack/files/hubblestack_nova/openssl.py b/hubblestack/files/hubblestack_nova/openssl.py index 98ee18948..c9f8f8bf4 100644 --- a/hubblestack/files/hubblestack_nova/openssl.py +++ b/hubblestack/files/hubblestack_nova/openssl.py @@ -65,7 +65,7 @@ """ -from __future__ import absolute_import + import logging import fnmatch @@ -174,7 +174,7 @@ def audit(data_list, tags, labels, debug=True, **kwargs): def _merge_yaml(ret, data, profile=None): if 'openssl' not in ret: ret['openssl'] = [] - for key, val in data.get('openssl', {}).iteritems(): + for key, val in data.get('openssl', {}).items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['openssl'].append({key: val}) @@ -184,7 +184,7 @@ def _merge_yaml(ret, data, profile=None): def _get_tags(data): ret = {} for audit_dict in data.get('openssl', {}): - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): tags_dict = audit_data.get('data', {}) tag = tags_dict.pop('tag') if tag not in ret: diff --git a/hubblestack/files/hubblestack_nova/oval_scanner.py b/hubblestack/files/hubblestack_nova/oval_scanner.py index 38c66fd57..328939a47 100644 --- a/hubblestack/files/hubblestack_nova/oval_scanner.py +++ b/hubblestack/files/hubblestack_nova/oval_scanner.py @@ -40,7 +40,7 @@ This scanner currently only supports the Linux platform. """ -from __future__ import absolute_import + import xml.etree.ElementTree as ET import json @@ -90,7 +90,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): def parse_impact_report(report, local_pkgs, hubble_format, impacted_pkgs=[]): """Parse into Hubble friendly format""" - for key, value in report.items(): + for key, value in list(report.items()): pkg_desc = 'Vulnerable Package(s): ' for pkg in value['installed']: pkg_desc += '{0}-{1}, '.format(pkg['name'], pkg['version']) @@ -125,7 +125,7 @@ def get_impact_report(vulns, local_pkgs, distro_name): def build_impact(vulns, local_pkgs, distro_name, result={}): """Build impacts based on pkg comparisons""" logging.debug('build_impact') - for data in vulns.values(): + for data in list(vulns.values()): for pkg in data['pkg']: name = pkg['name'] ver = pkg['version'] @@ -152,7 +152,7 @@ def build_impact(vulns, local_pkgs, distro_name, result={}): def build_impact_report(impact, report={}): """Build a report based on impacts""" logging.debug('build_impact_report') - for adv, detail in impact.items(): + for adv, detail in list(impact.items()): if adv not in report: report[adv] = { 'updated_pkg': [], @@ -187,7 +187,7 @@ def create_vulns(oval_and_maps, vulns={}): logging.debug('create_vulns') id_maps = oval_and_maps[0] oval = oval_and_maps[1] - for definition, data in id_maps.items(): + for definition, data in list(id_maps.items()): if definition in oval['definitions']: vulns[definition] = oval['definitions'][definition] vulns[definition]['pkg'] = [] @@ -217,7 +217,7 @@ def create_vulns(oval_and_maps, vulns={}): def map_oval_ids(oval, id_maps={}): """For every test, grab only tests with both state and obj references""" logging.debug('map_oval_ids') - for definition, data in oval['definitions'].items(): + for definition, data in list(oval['definitions'].items()): id_maps[definition] = {'objects': []} objects = id_maps[definition]['objects'] tests = data['tests'] diff --git a/hubblestack/files/hubblestack_nova/pkg.py b/hubblestack/files/hubblestack_nova/pkg.py index cc4771d1a..2470b0d89 100644 --- a/hubblestack/files/hubblestack_nova/pkg.py +++ b/hubblestack/files/hubblestack_nova/pkg.py @@ -56,7 +56,7 @@ trigger: state """ -from __future__ import absolute_import + import logging import fnmatch @@ -204,7 +204,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('pkg', {}): if topkey not in ret['pkg']: ret['pkg'][topkey] = [] - for key, val in data['pkg'][topkey].iteritems(): + for key, val in data['pkg'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['pkg'][topkey].append({key: val}) @@ -217,11 +217,11 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get('pkg', {}).iteritems(): + for toplist, toplevel in data.get('pkg', {}).items(): # pkg:blacklist for audit_dict in toplevel: # pkg:blacklist:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # pkg:blacklist:0:telnet tags_dict = audit_data.get('data', {}) # pkg:blacklist:0:telnet:data @@ -243,11 +243,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/pkgng_audit.py b/hubblestack/files/hubblestack_nova/pkgng_audit.py index de5faac18..c2bb73caa 100644 --- a/hubblestack/files/hubblestack_nova/pkgng_audit.py +++ b/hubblestack/files/hubblestack_nova/pkgng_audit.py @@ -2,7 +2,7 @@ """ Hubble Nova plugin for FreeBSD pkgng audit """ -from __future__ import absolute_import + import logging log = logging.getLogger(__name__) diff --git a/hubblestack/files/hubblestack_nova/service.py b/hubblestack/files/hubblestack_nova/service.py index 6dd57a774..1b1f2a3a5 100644 --- a/hubblestack/files/hubblestack_nova/service.py +++ b/hubblestack/files/hubblestack_nova/service.py @@ -50,7 +50,7 @@ trigger: state """ -from __future__ import absolute_import + import logging import fnmatch @@ -145,7 +145,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('service', {}): if topkey not in ret['service']: ret['service'][topkey] = [] - for key, val in data['service'][topkey].iteritems(): + for key, val in data['service'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['service'][topkey].append({key: val}) @@ -158,11 +158,11 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get('service', {}).iteritems(): + for toplist, toplevel in data.get('service', {}).items(): # service:blacklist for audit_dict in toplevel: # service:blacklist:0 - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # service:blacklist:0:telnet tags_dict = audit_data.get('data', {}) # service:blacklist:0:telnet:data @@ -184,11 +184,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): if tag not in ret: ret[tag] = [] formatted_data = {'name': name, diff --git a/hubblestack/files/hubblestack_nova/stat_nova.py b/hubblestack/files/hubblestack_nova/stat_nova.py index 8d2673b9a..ceb495c81 100644 --- a/hubblestack/files/hubblestack_nova/stat_nova.py +++ b/hubblestack/files/hubblestack_nova/stat_nova.py @@ -44,7 +44,7 @@ If the file exists, this setting is ignored. """ -from __future__ import absolute_import + import logging import os import fnmatch @@ -112,7 +112,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): if e in tag_data: expected[e] = tag_data[e] - if 'allow_more_strict' in expected.keys() and 'mode' not in expected.keys(): + if 'allow_more_strict' in list(expected.keys()) and 'mode' not in list(expected.keys()): reason_dict = {} reason = "'allow_more_strict' tag can't be specified without 'mode' tag." \ " Seems like a bug in hubble profile." @@ -129,7 +129,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): if not salt_ret: if not expected: ret['Success'].append(tag_data) - elif 'match_on_file_missing' in expected.keys() and expected['match_on_file_missing']: + elif 'match_on_file_missing' in list(expected.keys()) and expected['match_on_file_missing']: ret['Success'].append(tag_data) else: tag_data['failure_reason'] = "Could not get access any file at '{0}'. " \ @@ -140,7 +140,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): passed = True reason_dict = {} - for e in expected.keys(): + for e in list(expected.keys()): if e == 'allow_more_strict' or e == 'match_on_file_missing': continue r = salt_ret[e] @@ -149,7 +149,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): if r != '0': r = r[1:] allow_more_strict = False - if 'allow_more_strict' in expected.keys(): + if 'allow_more_strict' in list(expected.keys()): allow_more_strict = expected['allow_more_strict'] if not isinstance(allow_more_strict, bool): passed = False @@ -190,7 +190,7 @@ def _merge_yaml(ret, data, profile=None): """ if 'stat' not in ret: ret['stat'] = [] - for key, val in data.get('stat', {}).iteritems(): + for key, val in data.get('stat', {}).items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['stat'].append({key: val}) @@ -204,7 +204,7 @@ def _get_tags(data): ret = {} distro = __grains__.get('osfinger') for audit_dict in data.get('stat', []): - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): tags_dict = audit_data.get('data', {}) tags = None for osfinger in tags_dict: @@ -223,11 +223,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): if isinstance(tag, dict): tag_data = copy.deepcopy(tag) tag = tag_data.pop('tag') diff --git a/hubblestack/files/hubblestack_nova/sysctl.py b/hubblestack/files/hubblestack_nova/sysctl.py index bbcb302d7..38e44aeaf 100644 --- a/hubblestack/files/hubblestack_nova/sysctl.py +++ b/hubblestack/files/hubblestack_nova/sysctl.py @@ -26,7 +26,7 @@ trigger: state """ -from __future__ import absolute_import + import logging import fnmatch @@ -119,7 +119,7 @@ def _merge_yaml(ret, data, profile=None): """ if 'sysctl' not in ret: ret['sysctl'] = [] - for key, val in data.get('sysctl', {}).iteritems(): + for key, val in data.get('sysctl', {}).items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['sysctl'].append({key: val}) @@ -133,7 +133,7 @@ def _get_tags(data): ret = {} distro = __grains__.get('osfinger') for audit_dict in data.get('sysctl', []): - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): tags_dict = audit_data.get('data', {}) tags = None for osfinger in tags_dict: @@ -152,11 +152,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): if isinstance(tag, dict): tag_data = copy.deepcopy(tag) tag = tag_data.pop('tag') diff --git a/hubblestack/files/hubblestack_nova/systemctl.py b/hubblestack/files/hubblestack_nova/systemctl.py index 8adb1b500..d4c5f0ae0 100644 --- a/hubblestack/files/hubblestack_nova/systemctl.py +++ b/hubblestack/files/hubblestack_nova/systemctl.py @@ -27,7 +27,7 @@ """ -from __future__ import absolute_import + import logging import fnmatch @@ -122,7 +122,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get('systemctl', {}): if topkey not in ret['systemctl']: ret['systemctl'][topkey] = [] - for key, val in data['systemctl'][topkey].iteritems(): + for key, val in data['systemctl'][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret['systemctl'][topkey].append({key: val}) @@ -136,9 +136,9 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get('systemctl', {}).iteritems(): + for toplist, toplevel in data.get('systemctl', {}).items(): for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): tags_dict = audit_data.get('data', {}) tags = None for osfinger in tags_dict: @@ -158,11 +158,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/vulners_scanner.py b/hubblestack/files/hubblestack_nova/vulners_scanner.py index 651a55a38..cc8d218fa 100644 --- a/hubblestack/files/hubblestack_nova/vulners_scanner.py +++ b/hubblestack/files/hubblestack_nova/vulners_scanner.py @@ -17,7 +17,7 @@ This allows the module to run under a certain profile, as all of the other Nova modules do. """ -from __future__ import absolute_import + import logging import sys @@ -139,5 +139,5 @@ def _process_vulners(vulners): return [{'tag': 'Vulnerable package: {0}'.format(pkg), 'vulnerabilities': packages[pkg], - 'description': ', '.join(packages[pkg].keys())} + 'description': ', '.join(list(packages[pkg].keys()))} for pkg in packages] diff --git a/hubblestack/files/hubblestack_nova/win_auditpol.py b/hubblestack/files/hubblestack_nova/win_auditpol.py index 94d0fb660..cb15474c4 100644 --- a/hubblestack/files/hubblestack_nova/win_auditpol.py +++ b/hubblestack/files/hubblestack_nova/win_auditpol.py @@ -3,7 +3,7 @@ Windows auditpol audit module """ -from __future__ import absolute_import + import copy import csv import fnmatch @@ -121,7 +121,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -134,10 +134,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfullname') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -159,11 +159,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): @@ -190,7 +190,7 @@ def _auditpol_export(): return dump else: log.error('Nothing was returned from the auditpol command.') - except StandardError: + except Exception: log.error('An error occurred running the auditpol command.') diff --git a/hubblestack/files/hubblestack_nova/win_firewall.py b/hubblestack/files/hubblestack_nova/win_firewall.py index c84e50d7c..5555f9687 100644 --- a/hubblestack/files/hubblestack_nova/win_firewall.py +++ b/hubblestack/files/hubblestack_nova/win_firewall.py @@ -3,7 +3,7 @@ HubbleStack Nova Windows Firewall module """ -from __future__ import absolute_import + import copy import fnmatch import logging @@ -130,7 +130,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -143,10 +143,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfullname') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -168,11 +168,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): @@ -203,7 +203,7 @@ def _export_firewall(): return dump else: log.error('Nothing was returned from the auditpol command.') - except StandardError: + except Exception: log.error('An error occurred running the auditpol command.') diff --git a/hubblestack/files/hubblestack_nova/win_gp.py b/hubblestack/files/hubblestack_nova/win_gp.py index c1f0c0429..6c449b615 100644 --- a/hubblestack/files/hubblestack_nova/win_gp.py +++ b/hubblestack/files/hubblestack_nova/win_gp.py @@ -2,7 +2,7 @@ """ """ -from __future__ import absolute_import + import copy import fnmatch import logging @@ -102,7 +102,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -115,10 +115,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfinger') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -140,11 +140,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/win_pkg.py b/hubblestack/files/hubblestack_nova/win_pkg.py index 64e6a3b2c..874e88ad7 100644 --- a/hubblestack/files/hubblestack_nova/win_pkg.py +++ b/hubblestack/files/hubblestack_nova/win_pkg.py @@ -2,7 +2,7 @@ """ Windows package audit module """ -from __future__ import absolute_import + import copy import fnmatch @@ -126,7 +126,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -139,10 +139,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfullname') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -164,11 +164,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/win_reg.py b/hubblestack/files/hubblestack_nova/win_reg.py index f041ab6e4..9a3cfa9e9 100644 --- a/hubblestack/files/hubblestack_nova/win_reg.py +++ b/hubblestack/files/hubblestack_nova/win_reg.py @@ -2,7 +2,7 @@ """ """ -from __future__ import absolute_import + import copy import fnmatch import logging @@ -99,7 +99,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs): current = _find_option_value_in_reg(reg_dict['hive'], reg_dict['key'], reg_dict['value']) if isinstance(current, dict): tag_data['value_found'] = current - if any(x is False for x in current.values()): + if any(x is False for x in list(current.values())): ret['Failure'].append(tag_data) else: answer_list = [] @@ -162,7 +162,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -175,10 +175,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfullname') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -200,11 +200,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): diff --git a/hubblestack/files/hubblestack_nova/win_secedit.py b/hubblestack/files/hubblestack_nova/win_secedit.py index c03285796..eee7392e0 100644 --- a/hubblestack/files/hubblestack_nova/win_secedit.py +++ b/hubblestack/files/hubblestack_nova/win_secedit.py @@ -3,7 +3,7 @@ Windows secedit audit module """ -from __future__ import absolute_import + import copy import fnmatch import logging @@ -155,7 +155,7 @@ def _merge_yaml(ret, data, profile=None): if topkey in data.get(__virtualname__, {}): if topkey not in ret[__virtualname__]: ret[__virtualname__][topkey] = [] - for key, val in data[__virtualname__][topkey].iteritems(): + for key, val in data[__virtualname__][topkey].items(): if profile and isinstance(val, dict): val['nova_profile'] = profile ret[__virtualname__][topkey].append({key: val}) @@ -168,10 +168,10 @@ def _get_tags(data): """ ret = {} distro = __grains__.get('osfullname') - for toplist, toplevel in data.get(__virtualname__, {}).iteritems(): + for toplist, toplevel in data.get(__virtualname__, {}).items(): # secedit:whitelist for audit_dict in toplevel: - for audit_id, audit_data in audit_dict.iteritems(): + for audit_id, audit_data in audit_dict.items(): # secedit:whitelist:PasswordComplexity tags_dict = audit_data.get('data', {}) # secedit:whitelist:PasswordComplexity:data @@ -193,11 +193,11 @@ def _get_tags(data): if isinstance(tags, dict): # malformed yaml, convert to list of dicts tmp = [] - for name, tag in tags.iteritems(): + for name, tag in tags.items(): tmp.append({name: tag}) tags = tmp for item in tags: - for name, tag in item.iteritems(): + for name, tag in item.items(): tag_data = {} # Whitelist could have a dictionary, not a string if isinstance(tag, dict): @@ -228,7 +228,7 @@ def _secedit_export(): secedit_ret = _secedit_import(dump) ret = __salt__['file.remove'](dump) return secedit_ret - except StandardError: + except Exception: log.debug('Error occurred while trying to get / export secedit data') return False, None @@ -264,7 +264,7 @@ def _get_account_sid(): dict_return = {} lines = win32.split('\n') - lines = filter(None, lines) + lines = [_f for _f in lines if _f] if 'local:' in lines: lines.remove('local:') for line in lines: diff --git a/hubblestack/hangtime/__init__.py b/hubblestack/hangtime/__init__.py index 064dbd557..f0a2c89d9 100644 --- a/hubblestack/hangtime/__init__.py +++ b/hubblestack/hangtime/__init__.py @@ -5,7 +5,7 @@ # and, if applicable, load a fake timer wrapper assert signal.SIGALRM > 0 - from linux_itimers import HangTime, hangtime_wrapper + from hubblestack.hangtime.linux_itimers import HangTime, hangtime_wrapper except: - from fake import HangTime, hangtime_wrapper + from .fake import HangTime, hangtime_wrapper diff --git a/hubblestack/hec/dq.py b/hubblestack/hec/dq.py index cdec29f0f..9a8658727 100644 --- a/hubblestack/hec/dq.py +++ b/hubblestack/hec/dq.py @@ -84,7 +84,7 @@ def unlink_(self, fname): os.unlink(name) def decompress(self, dat): - if dat.startswith('BZ'): + if str(dat).startswith('BZ'): try: return bz2.BZ2Decompressor().decompress(dat) except IOError: @@ -131,6 +131,8 @@ def put(self, item, **meta): f = os.path.join(d, remainder) with open(f, 'wb') as fh: log.debug('writing item to disk cache') + if isinstance(bstr, str): + bstr = str.encode(bstr) fh.write(bstr) if meta: with open(f + '.meta', 'w') as fh: @@ -235,12 +237,12 @@ def files(self): """ generate all filenames in the diskqueue (returns iterable) """ def _k(x): try: - return [ int(i) for i in x.split('.') ] + return [int(i) for i in x.split('.')] except: pass return x for path, dirs, files in sorted(os.walk(self.directory)): - for fname in [ os.path.join(path, f) for f in sorted(files, key=_k) ]: + for fname in [os.path.join(path, f) for f in sorted(files, key=_k)]: if fname.endswith('.meta'): continue yield fname diff --git a/hubblestack/hec/opt.py b/hubblestack/hec/opt.py index 0f8eb3b77..acbb7d7d8 100644 --- a/hubblestack/hec/opt.py +++ b/hubblestack/hec/opt.py @@ -98,7 +98,7 @@ def _get_splunk_options(space, modality, **kw): else: final_opts[j] = opt[k] - if REQUIRED in final_opts.values(): + if REQUIRED in list(final_opts.values()): raise Exception('{0} must be specified in the {1} configs!'.format(req, space)) ret.append(final_opts) diff --git a/hubblestack/log.py b/hubblestack/log.py index 9d09d9fca..5c5a84d0e 100644 --- a/hubblestack/log.py +++ b/hubblestack/log.py @@ -3,7 +3,7 @@ Logging for the hubble daemon """ -from __future__ import print_function + import logging import time @@ -207,7 +207,7 @@ def filter_logs(opts_to_log, remove_dots=True): """ filtered_conf = _remove_sensitive_info(opts_to_log, PATTERNS_TO_FILTER) if remove_dots: - for key in filtered_conf.keys(): + for key in list(filtered_conf.keys()): if '.' in key: filtered_conf[key.replace('.', '_')] = filtered_conf.pop(key) return filtered_conf @@ -220,7 +220,7 @@ def _remove_sensitive_info(obj, patterns_to_filter): if isinstance(obj, dict): obj = { key: _remove_sensitive_info(value, patterns_to_filter) - for key, value in obj.iteritems() + for key, value in obj.items() if not any(patt in key for patt in patterns_to_filter)} elif isinstance(obj, list): obj = [_remove_sensitive_info(item, patterns_to_filter) diff --git a/hubblestack/splunklogging.py b/hubblestack/splunklogging.py index e2fdc0ea6..e81cb1537 100644 --- a/hubblestack/splunklogging.py +++ b/hubblestack/splunklogging.py @@ -78,7 +78,7 @@ def __init__(self): for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') - if isinstance(custom_field_value, (str, unicode)): + if isinstance(custom_field_value, str): event.update({custom_field_name: custom_field_value}) elif isinstance(custom_field_value, list): custom_field_value = ','.join(custom_field_value) diff --git a/hubblestack/status.py b/hubblestack/status.py index be1efa6a6..07334d1de 100644 --- a/hubblestack/status.py +++ b/hubblestack/status.py @@ -253,7 +253,7 @@ def mark(self, timestamp=None): timestamp = time.time() self = self.get_bucket(timestamp) else: - if isinstance(timestamp, (str, unicode)): + if isinstance(timestamp, str): timestamp = int(timestamp) self = self.get_bucket(timestamp) if timestamp < self.first_t: @@ -526,7 +526,7 @@ def short(cls, bucket=None): """ if bucket in ('*', 'all'): return [cls.short(b) for b in cls.buckets()] - return {k: v.asdict(bucket) for k, v in cls.dat.iteritems() if v.first_t > 0} + return {k: v.asdict(bucket) for k, v in cls.dat.items() if v.first_t > 0} @classmethod def as_json(cls, indent=2): diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index 6e8d0e8d5..f6f64677b 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -240,4 +240,4 @@ def pytest_sessionfinish(session, exitstatus): import subprocess p = subprocess.Popen(['chown', '-R', pcmf, sources_dir]) p.communicate() - print('\nchowned back files to {}'.format(pcmf)) + print(('\nchowned back files to {}'.format(pcmf))) diff --git a/tests/unittests/test_counters.py b/tests/unittests/test_counters.py index c09128ab0..e44a51bfe 100644 --- a/tests/unittests/test_counters.py +++ b/tests/unittests/test_counters.py @@ -133,7 +133,7 @@ def __enter__(self): max_buckets = self.kwargs.pop('max_buckets', 1e3) namespace = self.kwargs.pop('namespace', 'x') if self.kwargs: - raise ValueError('unknown arguments: {}', ', '.join(self.kwargs.keys())) + raise ValueError('unknown arguments: {}', ', '.join(list(self.kwargs.keys()))) opts = dict(bucket_len=bucket_len, max_buckets=max_buckets) # setup hubble_status diff --git a/tests/unittests/test_hec_dq.py b/tests/unittests/test_hec_dq.py index fb8ddfffe..6bb022bcf 100644 --- a/tests/unittests/test_hec_dq.py +++ b/tests/unittests/test_hec_dq.py @@ -8,7 +8,7 @@ @pytest.fixture def samp(): - return tuple(b'one two three four five'.split()) + return tuple('one two three four five'.split()) @pytest.fixture def dq(): @@ -17,9 +17,9 @@ def dq(): def test_disk_queue(dq): borked = False - dq.put(b'one', testinator=3) - dq.put(b'two', testinator=4) - dq.put(b'three', testinator=5) + dq.put('one', testinator=3) + dq.put('two', testinator=4) + dq.put('three', testinator=5) assert len(dq) == 13 assert dq.peek() == (b'one', {'testinator': 3}) @@ -27,21 +27,21 @@ def test_disk_queue(dq): assert dq.peek() == (b'two', {'testinator': 4}) assert len(dq) == 9 - assert dq.getz() == (b'two three', {'testinator': 5}) + assert dq.getz() == ('two three', {'testinator': 5}) assert len(dq) == 0 - dq.put(b'one') - dq.put(b'two') - dq.put(b'three') + dq.put('one') + dq.put('two') + dq.put('three') - assert dq.getz(8) == (b'one two', {}) - assert dq.getz(8) == (b'three', {}) + assert dq.getz(8) == ('one two', {}) + assert dq.getz(8) == ('three', {}) def _test_pop(samp,q): for i in samp: q.put(i) for i in samp: - assert q.peek() == (i, {}) + assert q.peek() == (str.encode(i), {}) q.pop() def test_dq_pop(samp,dq): diff --git a/tests/unittests/test_process.py b/tests/unittests/test_process.py index ec5f95f76..63aa9b50c 100644 --- a/tests/unittests/test_process.py +++ b/tests/unittests/test_process.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import + import os import pytest diff --git a/tests/unittests/test_process_status.py b/tests/unittests/test_process_status.py index 88309334b..97b6165ff 100644 --- a/tests/unittests/test_process_status.py +++ b/tests/unittests/test_process_status.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import + import mock import os @@ -18,8 +18,8 @@ def test_listProcesses_validReturn(self, mock_query): mock_query mock function for the `_run_query` function """ - mock_query.return_value = {'data': [{u'pid': u'123', u'name': u'foo'}, - {u'pid': u'321', u'name': u'bar'}], + mock_query.return_value = {'data': [{'pid': '123', 'name': 'foo'}, + {'pid': '321', 'name': 'bar'}], 'result': True} status, ret = hubblestack.extmods.fdg.process_status.list_processes() assert status @@ -43,14 +43,14 @@ def test__convertToStr_validDict_returnsValidDict(self): """ Test that when passed in valid data, it returns a dict with keys and values converted to string. """ - ret = hubblestack.extmods.fdg.process_status._convert_to_str([{u'pid': 123}, {'data': [1, 2, 3]}]) + ret = hubblestack.extmods.fdg.process_status._convert_to_str([{'pid': 123}, {'data': [1, 2, 3]}]) assert ret == [{'pid': '123'}, {'data': '[1, 2, 3]'}] def test__convertToStr_invalidArguments_returnsNone(self): """ Test that when passed in an invalid data type, the function returns none """ - ret = hubblestack.extmods.fdg.process_status._convert_to_str({u'pid': 123, 'data': [1, 2, 3]}) + ret = hubblestack.extmods.fdg.process_status._convert_to_str({'pid': 123, 'data': [1, 2, 3]}) assert ret is None ret = hubblestack.extmods.fdg.process_status._convert_to_str([123, 321, 'foo']) assert ret is None @@ -83,15 +83,15 @@ def test_findProcess_validArguments_returnsListOfDicts(self, mock_query): mock_query mock function for the `_run_query` function """ - mock_query.return_value = {'data': [{u'pid': u'123', u'name': u'bar'}, - {u'pid': u'321', u'name': u'foo'}], + mock_query.return_value = {'data': [{'pid': '123', 'name': 'bar'}, + {'pid': '321', 'name': 'foo'}], 'result': True} status, ret = hubblestack.extmods.fdg.process_status.find_process("state == 'S'") assert status assert ret == [{'pid': '123', 'name': 'bar'}, {'pid': '321', 'name': 'foo'}] - mock_query.return_value = {'data': [{u'pid': u'123', u'name': u'bar', u'parent': u'1', u'state': u'S'}, - {u'pid': u'321', u'name': u'foo', u'parent': u'1', u'state': u'S'}], + mock_query.return_value = {'data': [{'pid': '123', 'name': 'bar', 'parent': '1', 'state': 'S'}, + {'pid': '321', 'name': 'foo', 'parent': '1', 'state': 'S'}], 'result': True} status, ret = hubblestack.extmods.fdg.process_status.find_process("parent == 1 and state == 'S'", fields='parent,state') @@ -118,7 +118,7 @@ def test_isRunning_invalidArguments_returnsNone(self, mock_query): assert status is False assert ret is None # multiple processes returned by query - mock_query.return_value = {'data': [{u'state': 'S'}, {u'state': 'R'}]} + mock_query.return_value = {'data': [{'state': 'S'}, {'state': 'R'}]} status, ret = hubblestack.extmods.fdg.process_status.is_running('parent > 1') assert status is False assert ret is None @@ -137,12 +137,12 @@ def test_isRunning_validArguments_validReturn(self, mock_query): mock function for the `_run_query` function """ # process is running - mock_query.return_value = {'data': [{u'state': u'R'}]} + mock_query.return_value = {'data': [{'state': 'R'}]} status, ret = hubblestack.extmods.fdg.process_status.is_running('pid == 123') assert status assert ret # process is not running - mock_query.return_value = {'data': [{u'state': u'S'}]} + mock_query.return_value = {'data': [{'state': 'S'}]} status, ret = hubblestack.extmods.fdg.process_status.is_running("name == 'foo'") assert status assert ret is False @@ -173,8 +173,8 @@ def test_findChildren_validArguments_validReturn(self, mock_query): mock_query mock function for the `_run_query` function """ - mock_query.return_value = {'data': [{u'pid': u'123', u'name': u'foo', u'uid': u'123'}, - {u'pid': u'321', u'name': u'bar', u'uid': u'123'}], + mock_query.return_value = {'data': [{'pid': '123', 'name': 'foo', 'uid': '123'}, + {'pid': '321', 'name': 'bar', 'uid': '123'}], 'result': True} status, ret = hubblestack.extmods.fdg.process_status.find_children('foo', returned_fields='uid') assert status diff --git a/tests/unittests/test_pulsar.py b/tests/unittests/test_pulsar.py index 2d35c80f8..c268f40dd 100644 --- a/tests/unittests/test_pulsar.py +++ b/tests/unittests/test_pulsar.py @@ -218,9 +218,9 @@ def mk_more_files(self, count=1, to_write='supz-{0}\n'): def test_listify_anything(self): la = pulsar.PulsarWatchManager._listify_anything def lla(x,e): - assert len( la(x) ) == e + assert len(la(x)) == e def sla(x,e): - assert str(sorted(la(x))) == str(sorted(e)) + assert str(sorted(la(x), key=lambda x: str(x))) == str(sorted(e, key=lambda x: str(x))) lla(None, 0) lla([None], 0) lla(set([None]), 0) diff --git a/tests/unittests/test_readfile.py b/tests/unittests/test_readfile.py index 621c0e132..5b7034732 100644 --- a/tests/unittests/test_readfile.py +++ b/tests/unittests/test_readfile.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import + import json import os diff --git a/tests/unittests/test_time_sync.py b/tests/unittests/test_time_sync.py index e26e7d82a..455fe22f1 100644 --- a/tests/unittests/test_time_sync.py +++ b/tests/unittests/test_time_sync.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import + import mock import os diff --git a/utils/win_yaml_updater.py b/utils/win_yaml_updater.py index 5b2d1eb75..529b29216 100644 --- a/utils/win_yaml_updater.py +++ b/utils/win_yaml_updater.py @@ -1,5 +1,5 @@ # Updates yaml with PDF Table of Contents. Create new file with changes to most recent file labeled with CIS document version -from __future__ import print_function + from glob import glob import os @@ -28,7 +28,7 @@ def ItemAdder(title, server, key, tag, match, vtype, desc,): print("There is more than 1 pdf in the folder") truefile = False while not truefile: - filename = input('Path to PDF: ') + filename = eval(input('Path to PDF: ')) if os.path.isfile(filename): truefile = True else: @@ -38,7 +38,7 @@ def ItemAdder(title, server, key, tag, match, vtype, desc,): else: truefile = False while not truefile: - filename = input('Path to PDF: ') + filename = eval(input('Path to PDF: ')) if os.path.isfile(filename): truefile = True else: @@ -80,7 +80,7 @@ def ItemAdder(title, server, key, tag, match, vtype, desc,): print("There is more than 1 pdf in the folder") truefile = False while not truefile: - yamlname = input('Path to YAML: ') + yamlname = eval(input('Path to YAML: ')) if os.path.isfile(filename): truefile = True else: @@ -90,7 +90,7 @@ def ItemAdder(title, server, key, tag, match, vtype, desc,): else: truefile = False while not truefile: - yamlname = input('Path to yaml: ') + yamlname = eval(input('Path to yaml: ')) if os.path.isfile(filename): truefile = True else: @@ -105,20 +105,20 @@ def ItemAdder(title, server, key, tag, match, vtype, desc,): # flatenize the yaml flat_yaml = {} -for toplist, toplevel in hubyaml.iteritems(): +for toplist, toplevel in hubyaml.items(): # toplist windows sections win_secedit, toplevel is data inside toplist - for audit_dict, audit_info in toplevel.iteritems(): + for audit_dict, audit_info in toplevel.items(): # audit_dict = blacklist & whitelist data inside each toplist, audit_info = title dictionary - for audit_title, audit_data1 in audit_info.iteritems(): + for audit_title, audit_data1 in audit_info.items(): # audit_title is title of the check, audit_data is data dictionary audit_data = audit_data1.get('data', {}) audit_description = audit_data1.get('description', {}) if '(l1)' in audit_description.lower(): audit_description = audit_description[4:] - for audit_osfinger, audit_key1 in audit_data.iteritems(): + for audit_osfinger, audit_key1 in audit_data.items(): # osfinger server version for audit_other1 in audit_key1: - for audit_key, audit_other in audit_other1.iteritems(): + for audit_key, audit_other in audit_other1.items(): # flatenize! stag = audit_other['tag'].replace('CIS-', '') flat_yaml[stag] = {'value_type': audit_other['value_type'], 'match_output': audit_other['match_output'], 'section': toplist, 'tlist': audit_dict, 'check_title': audit_title, 'description': audit_description, 'os': audit_osfinger, 'audit_key': audit_key}