Skip to content

Commit

Permalink
🎉 Add --custom-plugins feature
Browse files Browse the repository at this point in the history 
- 🐍 Add add_shared_arguments() function in usage.py for DRYness
- 🐛 Fix issue #242 via passing `should_verify_secrets=not args.no_verify` to `from_parser_builder` call
- 🐛 Fix sorting issue in format_baseline_for_output() where --update and regular scan had different secret order
- 💯 All non-separated out files again :D

🙈 Hacks located in `def parse_args` of usage.py
  • Loading branch information
@KevinHock
KevinHock committed Oct 18, 2019
1 parent cf144db commit 1afce7b
Show file tree
Hide file tree
Showing 17 changed files with 499 additions and 211 deletions.
135 changes: 91 additions & 44 deletions detect_secrets/core/audit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class RedundantComparisonError(Exception):
'config': {},
}
EMPTY_STATS_RESULT = {
'signal': 0,
'signal': '0.00%',
'true-positives': {
'count': 0,
'files': defaultdict(int),
Expand Down Expand Up @@ -94,11 +94,12 @@ def audit_baseline(baseline_filename):

try:
_print_context(
filename,
secret,
current_secret_index,
total_choices,
original_baseline['plugins_used'],
filename=filename,
secret=secret,
count=current_secret_index,
total=total_choices,
plugins_used=original_baseline['plugins_used'],
custom_plugin_paths=original_baseline['custom_plugin_paths'],
)
decision = _get_user_decision(can_step_back=secret_iterator.can_step_back())
except SecretNotFoundOnSpecifiedLineError:
Expand Down Expand Up @@ -177,6 +178,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename):
header = '{} {}'
if is_removed:
plugins_used = old_baseline['plugins_used']
custom_plugin_paths = old_baseline['custom_plugin_paths']
header = header.format(
colorize('Status:', AnsiColor.BOLD),
'>> {} <<'.format(
Expand All @@ -185,6 +187,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename):
)
else:
plugins_used = new_baseline['plugins_used']
custom_plugin_paths = new_baseline['custom_plugin_paths']
header = header.format(
colorize('Status:', AnsiColor.BOLD),
'>> {} <<'.format(
Expand All @@ -194,13 +197,14 @@ def compare_baselines(old_baseline_filename, new_baseline_filename):

try:
_print_context(
filename,
secret,
current_index,
total_reviews,
plugins_used,
filename=filename,
secret=secret,
count=current_index,
total=total_reviews,
plugins_used=plugins_used,
custom_plugin_paths=custom_plugin_paths,
additional_header_lines=header,
force=is_removed,
force_line_printing=is_removed,
)
decision = _get_user_decision(
can_step_back=secret_iterator.can_step_back(),
Expand Down Expand Up @@ -265,7 +269,9 @@ def determine_audit_results(baseline, baseline_path):
'stats': deepcopy(EMPTY_STATS_RESULT),
}

secret_type_to_plugin_name = get_mapping_from_secret_type_to_class_name()
secret_type_to_plugin_name = get_mapping_from_secret_type_to_class_name(
custom_plugin_paths=baseline['custom_plugin_paths'],
)

total = 0
for filename, secret in all_secrets:
Expand All @@ -276,7 +282,8 @@ def determine_audit_results(baseline, baseline_path):
try:
secret_info['plaintext'] = get_raw_secret_value(
secret=secret,
plugin_settings=baseline['plugins_used'],
plugins_used=baseline['plugins_used'],
custom_plugin_paths=baseline['custom_plugin_paths'],
file_handle=io.StringIO(file_contents),
filename=filename,
)
Expand All @@ -290,13 +297,13 @@ def determine_audit_results(baseline, baseline_path):
audit_results['stats'][audit_result]['count'] += 1
audit_results['stats'][audit_result]['files'][filename] += 1
total += 1
audit_results['stats']['signal'] = str(
audit_results['stats']['signal'] = '{:.2f}'.format(
(
audit_results['stats']['true-positives']['count']
float(audit_results['stats']['true-positives']['count'])
/
total
) * 100,
)[:4] + '%'
) + '%'

for plugin_config in baseline['plugins_used']:
plugin_name = plugin_config['name']
Expand Down Expand Up @@ -339,7 +346,14 @@ def print_audit_results(baseline_filename):
def _get_baseline_from_file(filename): # pragma: no cover
try:
with open(filename) as f:
return json.loads(f.read())
baseline = json.loads(f.read())

# Convert custom_plugin_paths to a tuple so it is later hashable
# We want this so we can @lru_cache functions called with this
if 'custom_plugin_paths' in baseline:
baseline['custom_plugin_paths'] = tuple(baseline['custom_plugin_paths'])

return baseline
except (IOError, json.decoder.JSONDecodeError):
print('Not a valid baseline file!', file=sys.stderr)
return
Expand Down Expand Up @@ -480,9 +494,10 @@ def _print_context( # pragma: no cover
secret,
count,
total,
plugin_settings,
plugins_used,
custom_plugin_paths,
additional_header_lines=None,
force=False,
force_line_printing=False,
):
"""
:type filename: str
Expand All @@ -497,15 +512,24 @@ def _print_context( # pragma: no cover
:type total: int
:param total: total number of secrets in baseline
:type plugin_settings: list
:param plugin_settings: plugins used to create baseline.
:type plugins_used: list
:param plugins_used: output of "plugins_used" in baseline. e.g.
>>> [
... {
... 'name': 'Base64HighEntropyString',
... 'base64_limit': 4.5,
... },
... ]
:type custom_plugin_paths: (str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:type additional_header_lines: str
:param additional_header_lines: any additional lines to add to the
header of the interactive audit display.
:type force: bool
:param force: if True, will print the lines of code even if it doesn't
:type force_line_printing: bool
:param force_line_printing: if True, will print the lines of code even if it doesn't
find the secret expected
:raises: SecretNotFoundOnSpecifiedLineError
Expand All @@ -530,10 +554,11 @@ def _print_context( # pragma: no cover
error_obj = None
try:
secret_with_context = _get_secret_with_context(
filename,
secret,
plugin_settings,
force=force,
filename=filename,
secret=secret,
plugins_used=plugins_used,
custom_plugin_paths=custom_plugin_paths,
force_line_printing=force_line_printing,
)
print(secret_with_context)
except SecretNotFoundOnSpecifiedLineError as e:
Expand Down Expand Up @@ -615,9 +640,10 @@ def _get_file_line(filename, line_number):
def _get_secret_with_context(
filename,
secret,
plugin_settings,
plugins_used,
custom_plugin_paths,
lines_of_context=5,
force=False,
force_line_printing=False,
):
"""
Displays the secret, with surrounding lines of code for better context.
Expand All @@ -628,15 +654,24 @@ def _get_secret_with_context(
:type secret: dict, PotentialSecret.json() format
:param secret: the secret listed in baseline
:type plugin_settings: list
:param plugin_settings: plugins used to create baseline.
:type plugins_used: list
:param plugins_used: output of "plugins_used" in baseline. e.g.
>>> [
... {
... 'name': 'Base64HighEntropyString',
... 'base64_limit': 4.5,
... },
... ]
:type custom_plugin_paths: (str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:type lines_of_context: int
:param lines_of_context: number of lines displayed before and after
secret.
:type force: bool
:param force: if True, will print the lines of code even if it doesn't
:type force_line_printing: bool
:param force_line_printing: if True, will print the lines of code even if it doesn't
find the secret expected
:raises: SecretNotFoundOnSpecifiedLineError
Expand All @@ -656,18 +691,19 @@ def _get_secret_with_context(
)

raw_secret_value = get_raw_secret_value(
secret,
plugin_settings,
io.StringIO(file_content),
filename,
secret=secret,
plugins_used=plugins_used,
custom_plugin_paths=custom_plugin_paths,
file_handle=io.StringIO(file_content),
filename=filename,
)

try:
snippet.highlight_line(raw_secret_value)
except ValueError:
raise SecretNotFoundOnSpecifiedLineError(secret['line_number'])
except SecretNotFoundOnSpecifiedLineError:
if not force:
if not force_line_printing:
raise

snippet.target_line = colorize(
Expand All @@ -680,16 +716,26 @@ def _get_secret_with_context(

def get_raw_secret_value(
secret,
plugin_settings,
plugins_used,
custom_plugin_paths,
file_handle,
filename,
):
"""
:type secret: dict
:param secret: see caller's docstring
:type plugin_settings: list
:param plugin_settings: see caller's docstring
:type plugins_used: list
:param plugins_used: output of "plugins_used" in baseline. e.g.
>>> [
... {
... 'name': 'Base64HighEntropyString',
... 'base64_limit': 4.5,
... },
... ]
:type custom_plugin_paths: (str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:type file_handle: file object
:param file_handle: Open handle to file where the secret is
Expand All @@ -699,8 +745,9 @@ def get_raw_secret_value(
as a means of comparing whether two secrets are equal.
"""
plugin = initialize.from_secret_type(
secret['type'],
plugin_settings,
secret_type=secret['type'],
plugins_used=plugins_used,
custom_plugin_paths=custom_plugin_paths,
)

plugin_secrets = plugin.analyze(file_handle, filename)
Expand Down
11 changes: 10 additions & 1 deletion detect_secrets/core/baseline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
def initialize(
path,
plugins,
custom_plugin_paths,
exclude_files_regex=None,
exclude_lines_regex=None,
word_list_file=None,
Expand All @@ -30,6 +31,9 @@ def initialize(
:type plugins: tuple of detect_secrets.plugins.base.BasePlugin
:param plugins: rules to initialize the SecretsCollection with.
:type custom_plugin_paths: (str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:type exclude_files_regex: str|None
:type exclude_lines_regex: str|None
Expand All @@ -45,6 +49,7 @@ def initialize(
"""
output = SecretsCollection(
plugins,
custom_plugin_paths=custom_plugin_paths,
exclude_files=exclude_files_regex,
exclude_lines=exclude_lines_regex,
word_list_file=word_list_file,
Expand Down Expand Up @@ -256,7 +261,11 @@ def format_baseline_for_output(baseline):
for filename, secret_list in baseline['results'].items():
baseline['results'][filename] = sorted(
secret_list,
key=lambda x: (x['line_number'], x['hashed_secret']),
key=lambda secret: (
secret['line_number'],
secret['hashed_secret'],
secret['type'],
),
)

return json.dumps(
Expand Down
18 changes: 15 additions & 3 deletions detect_secrets/core/secrets_collection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class SecretsCollection(object):
def __init__(
self,
plugins=(),
custom_plugin_paths=(),
exclude_files=None,
exclude_lines=None,
word_list_file=None,
Expand All @@ -29,6 +30,9 @@ def __init__(
:type plugins: tuple of detect_secrets.plugins.base.BasePlugin
:param plugins: rules to determine whether a string is a secret
:type custom_plugin_paths: (str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:type exclude_files: str|None
:param exclude_files: optional regex for ignored paths.
Expand All @@ -42,12 +46,14 @@ def __init__(
:param word_list_hash: optional iterated sha1 hash of the words in the word list.
"""
self.data = {}
self.version = VERSION

self.plugins = plugins
self.custom_plugin_paths = custom_plugin_paths
self.exclude_files = exclude_files
self.exclude_lines = exclude_lines
self.word_list_file = word_list_file
self.word_list_hash = word_list_hash
self.version = VERSION

@classmethod
def load_baseline_from_string(cls, string):
Expand Down Expand Up @@ -107,16 +113,22 @@ def load_baseline_from_dict(cls, data):
result.word_list_hash = data['word_list']['hash']

if result.word_list_file:
# Always ignore the given `data['word_list']['hash']`
# Always ignore the existing `data['word_list']['hash']`
# The difference will show whenever the word list changes
automaton, result.word_list_hash = build_automaton(result.word_list_file)

# In v0.12.8 the `--custom-plugins` option got added
result.custom_plugin_paths = ()
if 'custom_plugin_paths' in data:
result.custom_plugin_paths = tuple(data['custom_plugin_paths'])

plugins = []
for plugin in data['plugins_used']:
plugin_classname = plugin.pop('name')
plugins.append(
initialize.from_plugin_classname(
plugin_classname,
custom_plugin_paths=result.custom_plugin_paths,
exclude_lines_regex=result.exclude_lines,
automaton=automaton,
should_verify_secrets=False,
Expand Down Expand Up @@ -221,7 +233,6 @@ def scan_file(self, filename, filename_key=None):
:returns: boolean; though this value is only used for testing
"""

if not filename_key:
filename_key = filename

Expand Down Expand Up @@ -300,6 +311,7 @@ def format_for_baseline_output(self):
'file': self.word_list_file,
'hash': self.word_list_hash,
},
'custom_plugin_paths': self.custom_plugin_paths,
'plugins_used': plugins_used,
'results': results,
'version': self.version,
Expand Down
Loading

0 comments on commit 1afce7b

Please sign in to comment.