diff --git a/README.md b/README.md index a86e45893..3b5647f34 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ For a look at recent changes, please see [CHANGELOG.md](https://github.com/Yelp/ If you are looking to contribute, please see [CONTRIBUTING.md](https://github.com/Yelp/detect-secrets/blob/master/CONTRIBUTING.md). + ## Example Usage ### Setting Up a Baseline @@ -44,6 +45,7 @@ If you are looking to contribute, please see [CONTRIBUTING.md](https://github.co $ detect-secrets scan > .secrets.baseline ``` + ### pre-commit Hook ``` @@ -56,12 +58,14 @@ $ cat .pre-commit-config.yaml exclude: .*/tests/.* ``` + ### Auditing a Baseline ``` $ detect-secrets audit .secrets.baseline ``` + ### Upgrading Baselines This is only applicable for upgrading baselines that have been created after version 0.9. @@ -71,6 +75,7 @@ For upgrading baselines lower than that version, just recreate it. $ detect-secrets scan --update .secrets.baseline ``` + ### Command Line `detect-secrets` is designed to be used as a git pre-commit hook, but you can also invoke `detect-secrets scan [path]` directly being `path` the file(s) and/or directory(ies) to scan (`path` defaults to `.` if not specified). @@ -93,6 +98,7 @@ either the client-side pre-commit hook, or the server-side secret scanner. 3. **Secrets Baseline**, to allowlist pre-existing secrets in the repository, so that they won't be continuously caught through scan iterations. + ### Client-side `pre-commit` Hook See [pre-commit](https://github.com/pre-commit/pre-commit) for instructions @@ -115,6 +121,7 @@ git diff --staged --name-only | xargs detect-secrets-hook Please see the [detect-secrets-server](https://github.com/Yelp/detect-secrets-server) repository for installation instructions. + ### Secrets Baseline ``` @@ -150,6 +157,7 @@ This may be a convenient way for you to allowlist secrets, without having to regenerate the entire baseline again. Furthermore, this makes the allowlisted secrets easily searchable, auditable, and maintainable. + ## Currently Supported Plugins The current heuristic searches we implement out of the box include: @@ -181,6 +189,9 @@ See [detect_secrets/ plugins](https://github.com/Yelp/detect-secrets/tree/master/detect_secrets/plugins) for more details. +There is also a `--custom-plugins` option in which you can bring your own plugins, e.g. `detect-secrets scan --custom-plugins testing/custom_plugins_dir/ --custom-plugins testing/hippo_plugin.py`. + + ## Caveats This is not meant to be a sure-fire solution to prevent secrets from entering @@ -188,11 +199,13 @@ the codebase. Only proper developer education can truly do that. This pre-commit hook merely implements several heuristics to try and prevent obvious cases of committing secrets. + ### Things that won't be prevented * Multi-line secrets * Default passwords that don't trigger the `KeywordDetector` (e.g. `login = "hunter2"`) + ### Plugin Configuration One method that this package uses to find secrets is by searching for high diff --git a/detect_secrets/core/audit.py b/detect_secrets/core/audit.py index ec47d11b8..92d418b5a 100644 --- a/detect_secrets/core/audit.py +++ b/detect_secrets/core/audit.py @@ -9,16 +9,16 @@ from copy import deepcopy from functools import lru_cache -from ..plugins.common import initialize -from ..plugins.common.util import get_mapping_from_secret_type_to_class_name -from ..util import get_git_remotes -from ..util import get_git_sha -from .baseline import merge_results -from .bidirectional_iterator import BidirectionalIterator -from .code_snippet import CodeSnippetHighlighter -from .color import AnsiColor -from .color import colorize -from .common import write_baseline_to_file +from detect_secrets.core.baseline import merge_results +from detect_secrets.core.bidirectional_iterator import BidirectionalIterator +from detect_secrets.core.code_snippet import CodeSnippetHighlighter +from detect_secrets.core.color import AnsiColor +from detect_secrets.core.color import colorize +from detect_secrets.core.common import write_baseline_to_file +from detect_secrets.plugins.common import initialize +from detect_secrets.plugins.common.util import get_mapping_from_secret_type_to_class_name +from detect_secrets.util import get_git_remotes +from detect_secrets.util import get_git_sha class SecretNotFoundOnSpecifiedLineError(Exception): @@ -48,7 +48,7 @@ class RedundantComparisonError(Exception): 'config': {}, } EMPTY_STATS_RESULT = { - 'signal': 0, + 'signal': '0.00%', 'true-positives': { 'count': 0, 'files': defaultdict(int), @@ -87,11 +87,12 @@ def audit_baseline(baseline_filename): try: _print_context( - filename, - secret, - current_secret_index, - total_choices, - original_baseline['plugins_used'], + filename=filename, + secret=secret, + count=current_secret_index, + total=total_choices, + plugins_used=original_baseline['plugins_used'], + custom_plugin_paths=original_baseline['custom_plugin_paths'], ) decision = _get_user_decision(can_step_back=secret_iterator.can_step_back()) except SecretNotFoundOnSpecifiedLineError: @@ -144,7 +145,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename): This means that we won't have cases where secrets are moved around; only added or removed. - NOTE: We don't want to do a version check, because we want to be able to + Note: We don't want to do a version check, because we want to be able to use this functionality across versions (to see how the new version fares compared to the old one). """ @@ -170,6 +171,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename): header = '{} {}' if is_removed: plugins_used = old_baseline['plugins_used'] + custom_plugin_paths = old_baseline['custom_plugin_paths'] header = header.format( colorize('Status:', AnsiColor.BOLD), '>> {} <<'.format( @@ -178,6 +180,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename): ) else: plugins_used = new_baseline['plugins_used'] + custom_plugin_paths = new_baseline['custom_plugin_paths'] header = header.format( colorize('Status:', AnsiColor.BOLD), '>> {} <<'.format( @@ -187,13 +190,14 @@ def compare_baselines(old_baseline_filename, new_baseline_filename): try: _print_context( - filename, - secret, - current_index, - total_reviews, - plugins_used, + filename=filename, + secret=secret, + count=current_index, + total=total_reviews, + plugins_used=plugins_used, + custom_plugin_paths=custom_plugin_paths, additional_header_lines=header, - force=is_removed, + force_line_printing=is_removed, ) decision = _get_user_decision( can_step_back=secret_iterator.can_step_back(), @@ -258,7 +262,9 @@ def determine_audit_results(baseline, baseline_path): 'stats': deepcopy(EMPTY_STATS_RESULT), } - secret_type_to_plugin_name = get_mapping_from_secret_type_to_class_name() + secret_type_to_plugin_name = get_mapping_from_secret_type_to_class_name( + custom_plugin_paths=baseline['custom_plugin_paths'], + ) total = 0 for filename, secret in all_secrets: @@ -269,7 +275,8 @@ def determine_audit_results(baseline, baseline_path): try: secret_info['plaintext'] = get_raw_secret_value( secret=secret, - plugin_settings=baseline['plugins_used'], + plugins_used=baseline['plugins_used'], + custom_plugin_paths=baseline['custom_plugin_paths'], file_handle=io.StringIO(file_contents), filename=filename, ) @@ -283,13 +290,13 @@ def determine_audit_results(baseline, baseline_path): audit_results['stats'][audit_result]['count'] += 1 audit_results['stats'][audit_result]['files'][filename] += 1 total += 1 - audit_results['stats']['signal'] = str( + audit_results['stats']['signal'] = '{:.2f}%'.format( ( float(audit_results['stats']['true-positives']['count']) / total ) * 100, - )[:4] + '%' + ) for plugin_config in baseline['plugins_used']: plugin_name = plugin_config['name'] @@ -332,7 +339,10 @@ def print_audit_results(baseline_filename): def _get_baseline_from_file(filename): # pragma: no cover try: with open(filename) as f: - return json.loads(f.read()) + baseline = json.loads(f.read()) + if 'custom_plugin_paths' in baseline: + baseline['custom_plugin_paths'] = tuple(baseline['custom_plugin_paths']) + return baseline except (IOError, json.decoder.JSONDecodeError): print('Not a valid baseline file!', file=sys.stderr) return @@ -473,9 +483,10 @@ def _print_context( # pragma: no cover secret, count, total, - plugin_settings, + plugins_used, + custom_plugin_paths, additional_header_lines=None, - force=False, + force_line_printing=False, ): """ :type filename: str @@ -490,15 +501,24 @@ def _print_context( # pragma: no cover :type total: int :param total: total number of secrets in baseline - :type plugin_settings: list - :param plugin_settings: plugins used to create baseline. + :type plugins_used: list + :param plugins_used: output of "plugins_used" in baseline. e.g. + >>> [ + ... { + ... 'name': 'Base64HighEntropyString', + ... 'base64_limit': 4.5, + ... }, + ... ] + + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. :type additional_header_lines: str :param additional_header_lines: any additional lines to add to the header of the interactive audit display. - :type force: bool - :param force: if True, will print the lines of code even if it doesn't + :type force_line_printing: bool + :param force_line_printing: if True, will print the lines of code even if it doesn't find the secret expected :raises: SecretNotFoundOnSpecifiedLineError @@ -523,10 +543,11 @@ def _print_context( # pragma: no cover error_obj = None try: secret_with_context = _get_secret_with_context( - filename, - secret, - plugin_settings, - force=force, + filename=filename, + secret=secret, + plugins_used=plugins_used, + custom_plugin_paths=custom_plugin_paths, + force_line_printing=force_line_printing, ) print(secret_with_context) except SecretNotFoundOnSpecifiedLineError as e: @@ -608,9 +629,10 @@ def _get_file_line(filename, line_number): def _get_secret_with_context( filename, secret, - plugin_settings, + plugins_used, + custom_plugin_paths, lines_of_context=5, - force=False, + force_line_printing=False, ): """ Displays the secret, with surrounding lines of code for better context. @@ -621,15 +643,24 @@ def _get_secret_with_context( :type secret: dict, PotentialSecret.json() format :param secret: the secret listed in baseline - :type plugin_settings: list - :param plugin_settings: plugins used to create baseline. + :type plugins_used: list + :param plugins_used: output of "plugins_used" in baseline. e.g. + >>> [ + ... { + ... 'name': 'Base64HighEntropyString', + ... 'base64_limit': 4.5, + ... }, + ... ] + + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. :type lines_of_context: int :param lines_of_context: number of lines displayed before and after secret. - :type force: bool - :param force: if True, will print the lines of code even if it doesn't + :type force_line_printing: bool + :param force_line_printing: if True, will print the lines of code even if it doesn't find the secret expected :raises: SecretNotFoundOnSpecifiedLineError @@ -649,10 +680,11 @@ def _get_secret_with_context( ) raw_secret_value = get_raw_secret_value( - secret, - plugin_settings, - io.StringIO(file_content), - filename, + secret=secret, + plugins_used=plugins_used, + custom_plugin_paths=custom_plugin_paths, + file_handle=io.StringIO(file_content), + filename=filename, ) try: @@ -660,7 +692,7 @@ def _get_secret_with_context( except ValueError: raise SecretNotFoundOnSpecifiedLineError(secret['line_number']) except SecretNotFoundOnSpecifiedLineError: - if not force: + if not force_line_printing: raise snippet.target_line = colorize( @@ -673,7 +705,8 @@ def _get_secret_with_context( def get_raw_secret_value( secret, - plugin_settings, + plugins_used, + custom_plugin_paths, file_handle, filename, ): @@ -681,8 +714,17 @@ def get_raw_secret_value( :type secret: dict :param secret: see caller's docstring - :type plugin_settings: list - :param plugin_settings: see caller's docstring + :type plugins_used: list + :param plugins_used: output of "plugins_used" in baseline. e.g. + >>> [ + ... { + ... 'name': 'Base64HighEntropyString', + ... 'base64_limit': 4.5, + ... }, + ... ] + + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. :type file_handle: file object :param file_handle: Open handle to file where the secret is @@ -692,8 +734,9 @@ def get_raw_secret_value( as a means of comparing whether two secrets are equal. """ plugin = initialize.from_secret_type( - secret['type'], - plugin_settings, + secret_type=secret['type'], + plugins_used=plugins_used, + custom_plugin_paths=custom_plugin_paths, ) plugin_secrets = plugin.analyze(file_handle, filename) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 4a6cf77bb..d74b7dbaf 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -14,6 +14,7 @@ def initialize( path, plugins, + custom_plugin_paths, exclude_files_regex=None, exclude_lines_regex=None, word_list_file=None, @@ -28,6 +29,9 @@ def initialize( :type plugins: tuple of detect_secrets.plugins.base.BasePlugin :param plugins: rules to initialize the SecretsCollection with. + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. + :type exclude_files_regex: str|None :type exclude_lines_regex: str|None @@ -43,6 +47,7 @@ def initialize( """ output = SecretsCollection( plugins, + custom_plugin_paths=custom_plugin_paths, exclude_files=exclude_files_regex, exclude_lines=exclude_lines_regex, word_list_file=word_list_file, @@ -125,7 +130,7 @@ def get_secrets_not_in_baseline(results, baseline): def trim_baseline_of_removed_secrets(results, baseline, filelist): """ - NOTE: filelist is not a comprehensive list of all files in the repo + Note: filelist is not a comprehensive list of all files in the repo (because we can't be sure whether --all-files is passed in as a parameter to pre-commit). @@ -254,7 +259,11 @@ def format_baseline_for_output(baseline): for filename, secret_list in baseline['results'].items(): baseline['results'][filename] = sorted( secret_list, - key=lambda x: (x['line_number'], x['hashed_secret']), + key=lambda secret: ( + secret['line_number'], + secret['hashed_secret'], + secret['type'], + ), ) return json.dumps( diff --git a/detect_secrets/core/code_snippet.py b/detect_secrets/core/code_snippet.py index 8d8fc0acb..30ae43a02 100644 --- a/detect_secrets/core/code_snippet.py +++ b/detect_secrets/core/code_snippet.py @@ -1,7 +1,7 @@ import itertools -from .color import AnsiColor -from .color import colorize +from detect_secrets.core.color import AnsiColor +from detect_secrets.core.color import colorize class CodeSnippetHighlighter: diff --git a/detect_secrets/core/common.py b/detect_secrets/core/common.py index 17815e651..8432225b3 100644 --- a/detect_secrets/core/common.py +++ b/detect_secrets/core/common.py @@ -1,4 +1,4 @@ -from .baseline import format_baseline_for_output +from detect_secrets.core.baseline import format_baseline_for_output def write_baseline_to_file(filename, data): diff --git a/detect_secrets/core/constants.py b/detect_secrets/core/constants.py index 812f9353a..01a80f741 100644 --- a/detect_secrets/core/constants.py +++ b/detect_secrets/core/constants.py @@ -2,7 +2,7 @@ # We don't scan files with these extensions. -# NOTE: We might be able to do this better with +# Note: We might be able to do this better with # `subprocess.check_output(['file', filename])` # and look for "ASCII text", but that might be more expensive. # diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index bf597e2ba..a8fca2d8c 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -59,7 +59,7 @@ def __init__( def set_secret(self, secret): self.secret_hash = self.hash_secret(secret) - # NOTE: Originally, we never wanted to keep the secret value in memory, + # Note: Originally, we never wanted to keep the secret value in memory, # after finding it in the codebase. However, to support verifiable # secrets (and avoid the pain of re-scanning again), we need to # keep the plaintext in memory as such. diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index 2ba5f8bf0..fafee55e0 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -18,6 +18,7 @@ class SecretsCollection: def __init__( self, plugins=(), + custom_plugin_paths=None, exclude_files=None, exclude_lines=None, word_list_file=None, @@ -27,6 +28,9 @@ def __init__( :type plugins: tuple of detect_secrets.plugins.base.BasePlugin :param plugins: rules to determine whether a string is a secret + :type custom_plugin_paths: Tuple[str]|None + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. + :type exclude_files: str|None :param exclude_files: optional regex for ignored paths. @@ -40,12 +44,14 @@ def __init__( :param word_list_hash: optional iterated sha1 hash of the words in the word list. """ self.data = {} + self.version = VERSION + self.plugins = plugins + self.custom_plugin_paths = custom_plugin_paths or () self.exclude_files = exclude_files self.exclude_lines = exclude_lines self.word_list_file = word_list_file self.word_list_hash = word_list_hash - self.version = VERSION @classmethod def load_baseline_from_string(cls, string): @@ -105,16 +111,20 @@ def load_baseline_from_dict(cls, data): result.word_list_hash = data['word_list']['hash'] if result.word_list_file: - # Always ignore the given `data['word_list']['hash']` + # Always ignore the existing `data['word_list']['hash']` # The difference will show whenever the word list changes automaton, result.word_list_hash = build_automaton(result.word_list_file) + # In v0.13.2 the `--custom-plugins` option got added + result.custom_plugin_paths = data.get('custom_plugin_paths', ()) + plugins = [] for plugin in data['plugins_used']: plugin_classname = plugin.pop('name') plugins.append( initialize.from_plugin_classname( plugin_classname, + custom_plugin_paths=result.custom_plugin_paths, exclude_lines_regex=result.exclude_lines, automaton=automaton, should_verify_secrets=False, @@ -259,7 +269,7 @@ def get_secret(self, filename, secret, type_=None): return None - # NOTE: We can only optimize this, if we knew the type of secret. + # Note: We can only optimize this, if we knew the type of secret. # Otherwise, we need to iterate through the set and find out. for obj in self.data[filename]: if obj.secret_hash == secret: @@ -293,6 +303,7 @@ def format_for_baseline_output(self): 'file': self.word_list_file, 'hash': self.word_list_hash, }, + 'custom_plugin_paths': self.custom_plugin_paths, 'plugins_used': plugins_used, 'results': results, 'version': self.version, diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index 545c73971..eacbf71b4 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -1,5 +1,7 @@ import argparse +import os from collections import namedtuple +from functools import lru_cache from detect_secrets import VERSION from detect_secrets.plugins.common.util import import_plugins @@ -25,6 +27,42 @@ def add_word_list_argument(parser): ) +def _is_valid_path(path): # pragma: no cover + if not os.path.exists(path): + raise argparse.ArgumentTypeError( + 'Invalid path: {}'.format(path), + ) + + return path + + +class TupleAction(argparse.Action): + def __call__(self, parser, namespace, values, options_string=None): + existing_values = getattr( + namespace, + self.dest, + ) + setattr( + namespace, + self.dest, + existing_values + (values,), + ) + + +def add_custom_plugins_argument(parser): + parser.add_argument( + '--custom-plugins', + action=TupleAction, + default=(), + dest='custom_plugin_paths', + help=( + 'Custom plugin Python files, or directories containing them. ' + 'Directories are not searched recursively.' + ), + type=_is_valid_path, + ) + + def add_use_all_plugins_argument(parser): parser.add_argument( '--use-all-plugins', @@ -42,6 +80,36 @@ def add_no_verify_flag(parser): ) +def add_shared_arguments(parser): + """ + These are arguments that are for both + `detect-secrets-hook` and `detect-secrets` console scripts. + """ + add_exclude_lines_argument(parser) + add_word_list_argument(parser) + add_custom_plugins_argument(parser) + add_use_all_plugins_argument(parser) + add_no_verify_flag(parser) + + +def get_parser_to_add_opt_out_options_to(parser): + """ + The pre-commit hook gets e.g. `--no-jwt-scan` type options + as well as the subparser for `detect-secrets scan`. + + :rtype: argparse.ArgumentParser + :returns: argparse.ArgumentParser to pass into PluginOptions + """ + if parser.prog == 'detect-secrets-hook': + return parser + + for action in parser._actions: # pragma: no cover (Always returns) + if isinstance(action, argparse._SubParsersAction): + for subparser in action.choices.values(): + if subparser.prog.endswith('scan'): + return subparser + + class ParserBuilder: def __init__(self): @@ -57,10 +125,7 @@ def add_pre_commit_arguments(self): self._add_filenames_argument()\ ._add_set_baseline_argument()\ - add_exclude_lines_argument(self.parser) - add_word_list_argument(self.parser) - add_use_all_plugins_argument(self.parser) - add_no_verify_flag(self.parser) + add_shared_arguments(self.parser) PluginOptions(self.parser).add_arguments() @@ -77,10 +142,39 @@ def add_console_use_arguments(self): return self def parse_args(self, argv): - output = self.parser.parse_args(argv) - PluginOptions.consolidate_args(output) + # We temporarily remove '--help' so that we can give the full + # amount of options (e.g. --no-custom-detector) after loading + # custom plugins. + argv_without_help = list( + filter( + lambda arg: ( + arg not in ('-h', '--help') + ), + argv, + ), + ) - return output + known_args, _ = self.parser.parse_known_args( + args=argv_without_help, + ) + + # Audit does not use the `--custom-plugins` argument + # It pulls custom_plugins from the audited baseline + if hasattr(known_args, 'custom_plugin_paths'): + # Add e.g. `--no-jwt-scan` type options + # now that we can use the --custom-plugins argument + PluginOptions( + get_parser_to_add_opt_out_options_to(self.parser), + ).add_opt_out_options( + known_args.custom_plugin_paths, + ) + + args = self.parser.parse_args( + args=argv, + ) + PluginOptions.consolidate_args(args) + + return args def _add_version_argument(self): self.parser.add_argument( @@ -144,10 +238,7 @@ def _add_initialize_baseline_argument(self): ), ) - # Pairing `--exclude-lines` and `--word-list` to - # both pre-commit and `--scan` because it can be used for both. - add_exclude_lines_argument(self.parser) - add_word_list_argument(self.parser) + add_shared_arguments(self.parser) # Pairing `--exclude-files` with `--scan` because it's only used for the initialization. # The pre-commit hook framework already has an `exclude` option that can @@ -168,18 +259,12 @@ def _add_initialize_baseline_argument(self): dest='import_filename', ) - # Pairing `--update` with `--use-all-plugins` to overwrite plugins list - # from baseline - add_use_all_plugins_argument(self.parser) - self.parser.add_argument( '--all-files', action='store_true', help='Scan all files recursively (as compared to only scanning git tracked files).', ) - add_no_verify_flag(self.parser) - return self def _add_adhoc_scanning_argument(self): @@ -305,13 +390,17 @@ def get_disabled_help_text(plugin): return 'Disables {}'.format(line) -class PluginOptions: - - all_plugins = [ +@lru_cache(maxsize=1) +def get_all_plugin_descriptors(custom_plugin_paths): + return [ PluginDescriptor.from_plugin_class(plugin, name) - for name, plugin in import_plugins().items() + for name, plugin in + import_plugins(custom_plugin_paths).items() ] + +class PluginOptions: + def __init__(self, parser): self.parser = parser.add_argument_group( title='plugins', @@ -324,7 +413,6 @@ def __init__(self, parser): def add_arguments(self): self._add_custom_limits() - self._add_opt_out_options() self._add_keyword_exclude() return self @@ -333,7 +421,7 @@ def add_arguments(self): def get_disabled_plugins(args): return [ plugin.classname - for plugin in PluginOptions.all_plugins + for plugin in get_all_plugin_descriptors(args.custom_plugin_paths) if plugin.classname not in args.plugins ] @@ -357,7 +445,7 @@ def consolidate_args(args): active_plugins = {} is_using_default_value = {} - for plugin in PluginOptions.all_plugins: + for plugin in get_all_plugin_descriptors(args.custom_plugin_paths): arg_name = PluginOptions._convert_flag_text_to_argument_name( plugin.disable_flag_text, ) @@ -410,8 +498,8 @@ def _add_custom_limits(self): help=high_entropy_help_text + 'defaults to 3.0.', ) - def _add_opt_out_options(self): - for plugin in self.all_plugins: + def add_opt_out_options(self, custom_plugin_paths): + for plugin in get_all_plugin_descriptors(custom_plugin_paths): self.parser.add_argument( plugin.disable_flag_text, action='store_true', diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 8a033e453..a942bc830 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -17,9 +17,9 @@ def parse_args(argv): .parse_args(argv) -def main(argv=None): +def main(argv=sys.argv[1:]): if len(sys.argv) == 1: # pragma: no cover - sys.argv.append('-h') + sys.argv.append('--help') args = parse_args(argv) if args.verbose: # pragma: no cover @@ -34,7 +34,8 @@ def main(argv=None): # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder( - args.plugins, + plugins_dict=args.plugins, + custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, @@ -142,8 +143,7 @@ def _perform_scan(args, plugins, automaton, word_list_hash): automaton=automaton, ) - # Favors `--exclude-files` and `--exclude-lines` CLI arguments - # over existing baseline's regexes (if given) + # Favors CLI arguments over existing baseline configuration if old_baseline: if not args.exclude_files: args.exclude_files = _get_exclude_files(old_baseline) @@ -160,18 +160,25 @@ def _perform_scan(args, plugins, automaton, word_list_hash): ): args.word_list_file = old_baseline['word_list']['file'] + if ( + not args.custom_plugin_paths + and old_baseline.get('custom_plugin_paths') + ): + args.custom_plugin_paths = old_baseline['custom_plugin_paths'] + # If we have knowledge of an existing baseline file, we should use # that knowledge and add it to our exclude_files regex. if args.import_filename: _add_baseline_to_exclude_files(args) new_baseline = baseline.initialize( + path=args.path, plugins=plugins, + custom_plugin_paths=args.custom_plugin_paths, exclude_files_regex=args.exclude_files, exclude_lines_regex=args.exclude_lines, word_list_file=args.word_list_file, word_list_hash=word_list_hash, - path=args.path, should_scan_all_files=args.all_files, ).format_for_baseline_output() diff --git a/detect_secrets/plugins/artifactory.py b/detect_secrets/plugins/artifactory.py index 0d0ee6a8d..743f76526 100644 --- a/detect_secrets/plugins/artifactory.py +++ b/detect_secrets/plugins/artifactory.py @@ -1,6 +1,6 @@ import re -from .base import RegexBasedDetector +from detect_secrets.plugins.base import RegexBasedDetector class ArtifactoryDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/aws.py b/detect_secrets/plugins/aws.py index cf0b86da2..e1f73619a 100644 --- a/detect_secrets/plugins/aws.py +++ b/detect_secrets/plugins/aws.py @@ -10,9 +10,9 @@ import requests -from .base import classproperty -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.base import RegexBasedDetector class AWSKeyDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index e64d85168..e911a3813 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -3,13 +3,13 @@ from abc import abstractmethod from abc import abstractproperty -from .common.constants import ALLOWLIST_REGEXES from detect_secrets.core.code_snippet import CodeSnippetHighlighter from detect_secrets.core.constants import VerifiedResult from detect_secrets.core.potential_secret import PotentialSecret +from detect_secrets.plugins.common.constants import ALLOWLIST_REGEXES -# NOTE: In this whitepaper (Section V-D), it suggests that there's an +# Note: In this whitepaper (Section V-D), it suggests that there's an # 80% chance of finding a multi-factor secret (e.g. username + # password) within five lines of context, before and after a secret. # @@ -162,7 +162,7 @@ def analyze_line(self, string, line_num, filename): :param filename: string; name of file being analyzed :returns: dictionary - NOTE: line_num and filename are used for PotentialSecret creation only. + Note: line_num and filename are used for PotentialSecret creation only. """ return self.analyze_string_content( string, @@ -178,7 +178,7 @@ def analyze_string_content(self, string, line_num, filename): :param filename: string; name of file being analyzed :returns: dictionary - NOTE: line_num and filename are used for PotentialSecret creation only. + Note: line_num and filename are used for PotentialSecret creation only. """ raise NotImplementedError diff --git a/detect_secrets/plugins/basic_auth.py b/detect_secrets/plugins/basic_auth.py index 5baeb7dd4..62cedaf13 100644 --- a/detect_secrets/plugins/basic_auth.py +++ b/detect_secrets/plugins/basic_auth.py @@ -1,6 +1,6 @@ import re -from .base import RegexBasedDetector +from detect_secrets.plugins.base import RegexBasedDetector # This list is derived from RFC 3986 Section 2.2. diff --git a/detect_secrets/plugins/cloudant.py b/detect_secrets/plugins/cloudant.py index 14192deec..a69a7b5c3 100644 --- a/detect_secrets/plugins/cloudant.py +++ b/detect_secrets/plugins/cloudant.py @@ -2,8 +2,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class CloudantDetector(RegexBasedDetector): @@ -89,7 +89,6 @@ def find_account(content): http=CloudantDetector.http, opt_basic_auth=opt_basic_auth, cl_account=account, - cl_api_key=CloudantDetector.cl_api_key, dot=CloudantDetector.dot, cloudant_api_url=CloudantDetector.cloudant_api_url, ), diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py index 7df4553b3..a09c27ec0 100644 --- a/detect_secrets/plugins/common/initialize.py +++ b/detect_secrets/plugins/common/initialize.py @@ -1,12 +1,13 @@ """Intelligent initialization of plugins.""" -from .util import get_mapping_from_secret_type_to_class_name -from .util import import_plugins from detect_secrets.core.log import log from detect_secrets.core.usage import PluginOptions +from detect_secrets.plugins.common.util import get_mapping_from_secret_type_to_class_name +from detect_secrets.plugins.common.util import import_plugins def from_parser_builder( plugins_dict, + custom_plugin_paths, exclude_lines_regex=None, automaton=None, should_verify_secrets=False, @@ -15,6 +16,9 @@ def from_parser_builder( :param plugins_dict: plugins dictionary received from ParserBuilder. See example in tests.core.usage_test. + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. + :type exclude_lines_regex: str|None :param exclude_lines_regex: optional regex for ignored lines. @@ -27,14 +31,15 @@ def from_parser_builder( """ output = [] - for plugin_name in plugins_dict: + for plugin_classname in plugins_dict: output.append( from_plugin_classname( - plugin_name, + plugin_classname, + custom_plugin_paths=custom_plugin_paths, exclude_lines_regex=exclude_lines_regex, automaton=automaton, should_verify_secrets=should_verify_secrets, - **plugins_dict[plugin_name] + **plugins_dict[plugin_classname] ), ) @@ -106,6 +111,7 @@ def _remove_key(d, key): return from_parser_builder( plugins_dict, + custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, @@ -137,13 +143,16 @@ def _remove_key(d, key): return from_parser_builder( plugins_dict, + custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, + should_verify_secrets=not args.no_verify, ) def from_plugin_classname( plugin_classname, + custom_plugin_paths, exclude_lines_regex=None, automaton=None, should_verify_secrets=False, @@ -154,6 +163,9 @@ def from_plugin_classname( :type plugin_classname: str :param plugin_classname: subclass of BasePlugin. + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. + :type exclude_lines_regex: str|None :param exclude_lines_regex: optional regex for ignored lines. @@ -163,14 +175,18 @@ def from_plugin_classname( :type should_verify_secrets: bool """ try: - klass = import_plugins()[plugin_classname] + klass = import_plugins(custom_plugin_paths)[plugin_classname] except KeyError: log.error('Error: No such `{}` plugin to initialize.'.format(plugin_classname)) log.error('Chances are you should run `pre-commit autoupdate`.') log.error( - 'This error occurs when using a baseline that was made by ' + 'This error can occur when using a baseline that was made by ' 'a newer detect-secrets version than the one running.', ) + log.error( + 'It can also occur if the baseline has custom plugin paths, ' + 'but the `--custom-plugins` option was not passed.', + ) raise TypeError try: @@ -181,41 +197,45 @@ def from_plugin_classname( **kwargs ) except TypeError: - log.warning('Unable to initialize plugin!') + log.error('Unable to initialize plugin!') raise return instance -def from_secret_type(secret_type, settings): +def from_secret_type(secret_type, plugins_used, custom_plugin_paths): """ Note: Only called from audit.py :type secret_type: str :param secret_type: unique identifier for plugin type - :type settings: list - :param settings: output of "plugins_used" in baseline. e.g. + :type plugins_used: list + :param plugins_used: output of "plugins_used" in baseline. e.g. >>> [ ... { ... 'name': 'Base64HighEntropyString', ... 'base64_limit': 4.5, ... }, ... ] + + :type custom_plugin_paths: Tuple[str] + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. """ - mapping = get_mapping_from_secret_type_to_class_name() + mapping = get_mapping_from_secret_type_to_class_name(custom_plugin_paths) try: classname = mapping[secret_type] except KeyError: return None - for plugin in settings: + for plugin in plugins_used: if plugin['name'] == classname: plugin_init_vars = plugin.copy() plugin_init_vars.pop('name') return from_plugin_classname( classname, + custom_plugin_paths=custom_plugin_paths, # `audit` does not need to # perform exclusion, filtering or verification diff --git a/detect_secrets/plugins/common/util.py b/detect_secrets/plugins/common/util.py index 9815b566f..321092533 100644 --- a/detect_secrets/plugins/common/util.py +++ b/detect_secrets/plugins/common/util.py @@ -1,53 +1,133 @@ +import importlib.util +import inspect import os from abc import abstractproperty from functools import lru_cache -from importlib import import_module from detect_secrets.plugins.base import BasePlugin from detect_secrets.util import get_root_directory @lru_cache(maxsize=1) -def get_mapping_from_secret_type_to_class_name(): - """Returns secret_type => plugin classname""" +def get_mapping_from_secret_type_to_class_name(custom_plugin_paths): + """Returns dictionary of secret_type => plugin classname""" return { plugin.secret_type: name - for name, plugin in import_plugins().items() + for name, plugin in import_plugins(custom_plugin_paths).items() } +def _dynamically_import_module(path_to_import, module_name): + """ + :type path_to_import: str + :type module_name: str + + :rtype: module + """ + spec = importlib.util.spec_from_file_location( + module_name, + path_to_import, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _is_valid_concrete_plugin_class(attr): + """ + :type attr: Any + + :rtype: bool + """ + return ( + inspect.isclass(attr) + and + issubclass(attr, BasePlugin) + and + # Heuristic to determine abstract classes + not isinstance(attr.secret_type, abstractproperty) + ) + + @lru_cache(maxsize=1) -def import_plugins(): +def import_plugins(custom_plugin_paths): """ + :type custom_plugin_paths: tuple(str,) + :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins. + :rtype: Dict[str, Type[TypeVar('Plugin', bound=BasePlugin)]] """ - modules = [] - for root, _, files in os.walk( - os.path.join(get_root_directory(), 'detect_secrets/plugins'), - ): - for filename in files: - if not filename.startswith('_'): - modules.append(os.path.splitext(filename)[0]) + path_and_module_name_pairs = [] - # Only want to import top level files - break + # Handle files + for path_to_import in custom_plugin_paths: + if os.path.isfile(path_to_import): + # [:-3] for removing '.py' + module_name = path_to_import[:-3].replace('/', '.') + path_and_module_name_pairs.append( + ( + path_to_import, + module_name, + ), + ) - plugins = {} - for module_name in modules: - module = import_module('detect_secrets.plugins.{}'.format(module_name)) - for name in filter(lambda x: not x.startswith('_'), dir(module)): - plugin = getattr(module, name) - try: - if not issubclass(plugin, BasePlugin): - continue - except TypeError: - # Occurs when plugin is not a class type. + # Handle directories + regular_plugins_dir = os.path.join( + get_root_directory(), + 'detect_secrets/plugins', + ) + plugin_dirs = ( + [regular_plugins_dir] + + + list( + filter( + lambda path: ( + os.path.isdir(path) + ), + custom_plugin_paths, + ), + ) + ) + for plugin_dir in plugin_dirs: + for filename in os.listdir( + plugin_dir, + ): + if ( + filename.startswith('_') + or not filename.endswith('.py') + ): continue - # Use this as a heuristic to determine abstract classes - if isinstance(plugin.secret_type, abstractproperty): - continue + path_to_import = os.path.join( + plugin_dir, + filename, + ) + + # [:-3] for removing '.py' + if plugin_dir == regular_plugins_dir: + module_name = 'detect_secrets.plugins.{}'.format(filename[:-3]) + else: + module_name = path_to_import[:-3].replace('/', '.') + path_and_module_name_pairs.append( + ( + path_to_import, + module_name, + ), + ) - plugins[name] = plugin + # Do the importing + plugins = {} + for path_to_import, module_name in path_and_module_name_pairs: + module = _dynamically_import_module( + path_to_import, + module_name, + ) + for attr_name in filter( + lambda attr_name: not attr_name.startswith('_'), + dir(module), + ): + attr = getattr(module, attr_name) + if _is_valid_concrete_plugin_class(attr): + plugins[attr_name] = attr return plugins diff --git a/detect_secrets/plugins/common/yaml_file_parser.py b/detect_secrets/plugins/common/yaml_file_parser.py index 65e3efc80..7b17fbae0 100644 --- a/detect_secrets/plugins/common/yaml_file_parser.py +++ b/detect_secrets/plugins/common/yaml_file_parser.py @@ -1,6 +1,6 @@ import yaml -from .constants import ALLOWLIST_REGEX +from detect_secrets.plugins.common.constants import ALLOWLIST_REGEX class YamlFileParser: diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py index cf78f90cb..f43553a8f 100644 --- a/detect_secrets/plugins/high_entropy_strings.py +++ b/detect_secrets/plugins/high_entropy_strings.py @@ -9,17 +9,17 @@ import yaml -from .base import BasePlugin -from .base import classproperty -from .common.filetype import determine_file_type -from .common.filetype import FileType -from .common.filters import get_aho_corasick_helper -from .common.filters import is_false_positive_with_line_context -from .common.filters import is_potential_uuid -from .common.filters import is_sequential_string -from .common.ini_file_parser import IniFileParser -from .common.yaml_file_parser import YamlFileParser from detect_secrets.core.potential_secret import PotentialSecret +from detect_secrets.plugins.base import BasePlugin +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.common.filetype import determine_file_type +from detect_secrets.plugins.common.filetype import FileType +from detect_secrets.plugins.common.filters import get_aho_corasick_helper +from detect_secrets.plugins.common.filters import is_false_positive_with_line_context +from detect_secrets.plugins.common.filters import is_potential_uuid +from detect_secrets.plugins.common.filters import is_sequential_string +from detect_secrets.plugins.common.ini_file_parser import IniFileParser +from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser class HighEntropyStringsPlugin(BasePlugin): @@ -146,7 +146,7 @@ def adhoc_scan(self, string): filename='does_not_matter', ) - # NOTE: Trailing space allows for nicer formatting + # Note: Trailing space allows for nicer formatting output = 'False' if not results else 'True ' if results: # We currently assume that there's at most one secret per line. diff --git a/detect_secrets/plugins/ibm_cloud_iam.py b/detect_secrets/plugins/ibm_cloud_iam.py index 65a23d116..71a7aff0b 100644 --- a/detect_secrets/plugins/ibm_cloud_iam.py +++ b/detect_secrets/plugins/ibm_cloud_iam.py @@ -1,7 +1,7 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class IbmCloudIamDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/ibm_cos_hmac.py b/detect_secrets/plugins/ibm_cos_hmac.py index 5849f844a..a306903a3 100644 --- a/detect_secrets/plugins/ibm_cos_hmac.py +++ b/detect_secrets/plugins/ibm_cos_hmac.py @@ -4,8 +4,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class IbmCosHmacDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/jwt.py b/detect_secrets/plugins/jwt.py index b16ed9083..32f67be91 100644 --- a/detect_secrets/plugins/jwt.py +++ b/detect_secrets/plugins/jwt.py @@ -5,8 +5,8 @@ import json import re -from .base import classproperty -from .base import RegexBasedDetector +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.base import RegexBasedDetector class JwtTokenDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index 42c2c7918..695f5f7e4 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -26,13 +26,13 @@ """ import re -from .base import BasePlugin -from .base import classproperty -from .common.filetype import determine_file_type -from .common.filetype import FileType -from .common.filters import get_aho_corasick_helper -from .common.filters import is_sequential_string from detect_secrets.core.potential_secret import PotentialSecret +from detect_secrets.plugins.base import BasePlugin +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.common.filetype import determine_file_type +from detect_secrets.plugins.common.filetype import FileType +from detect_secrets.plugins.common.filters import get_aho_corasick_helper +from detect_secrets.plugins.common.filters import is_sequential_string # Note: All values here should be lowercase @@ -216,7 +216,6 @@ denylist=DENYLIST_REGEX, nonWhitespace=OPTIONAL_NON_WHITESPACE, quote=QUOTE, - closing=CLOSING, whitespace=OPTIONAL_WHITESPACE, secret=SECRET, ), diff --git a/detect_secrets/plugins/mailchimp.py b/detect_secrets/plugins/mailchimp.py index f90f6cc90..768ee34cd 100644 --- a/detect_secrets/plugins/mailchimp.py +++ b/detect_secrets/plugins/mailchimp.py @@ -6,8 +6,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class MailchimpDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/private_key.py b/detect_secrets/plugins/private_key.py index edd4b5681..444fadd3a 100644 --- a/detect_secrets/plugins/private_key.py +++ b/detect_secrets/plugins/private_key.py @@ -26,7 +26,7 @@ """ import re -from .base import RegexBasedDetector +from detect_secrets.plugins.base import RegexBasedDetector class PrivateKeyDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/slack.py b/detect_secrets/plugins/slack.py index cb6427d89..15e494813 100644 --- a/detect_secrets/plugins/slack.py +++ b/detect_secrets/plugins/slack.py @@ -5,8 +5,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class SlackDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/softlayer.py b/detect_secrets/plugins/softlayer.py index b2ca78cf6..0da629df8 100644 --- a/detect_secrets/plugins/softlayer.py +++ b/detect_secrets/plugins/softlayer.py @@ -2,8 +2,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class SoftlayerDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/stripe.py b/detect_secrets/plugins/stripe.py index f7f1839b9..32c5e3b20 100644 --- a/detect_secrets/plugins/stripe.py +++ b/detect_secrets/plugins/stripe.py @@ -3,8 +3,8 @@ import requests -from .base import RegexBasedDetector from detect_secrets.core.constants import VerifiedResult +from detect_secrets.plugins.base import RegexBasedDetector class StripeDetector(RegexBasedDetector): diff --git a/detect_secrets/plugins/twilio.py b/detect_secrets/plugins/twilio.py index 4d5b397e3..57d0f1900 100644 --- a/detect_secrets/plugins/twilio.py +++ b/detect_secrets/plugins/twilio.py @@ -3,7 +3,7 @@ """ import re -from .base import RegexBasedDetector +from detect_secrets.plugins.base import RegexBasedDetector class TwilioKeyDetector(RegexBasedDetector): diff --git a/detect_secrets/pre_commit_hook.py b/detect_secrets/pre_commit_hook.py index b516e4822..cd5bf2308 100644 --- a/detect_secrets/pre_commit_hook.py +++ b/detect_secrets/pre_commit_hook.py @@ -17,13 +17,13 @@ log = get_logger(format_string='%(message)s') -def parse_args(argv): +def parse_args(argv): # pragma: no cover (Mocked) return ParserBuilder()\ .add_pre_commit_arguments()\ .parse_args(argv) -def main(argv=None): +def main(argv=sys.argv[1:]): args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) @@ -42,7 +42,8 @@ def main(argv=None): automaton, word_list_hash = build_automaton(args.word_list_file) plugins = initialize.from_parser_builder( - args.plugins, + plugins_dict=args.plugins, + custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, @@ -51,13 +52,14 @@ def main(argv=None): # Merge plugins from baseline if baseline_collection: plugins = initialize.merge_plugins_from_baseline( - baseline_collection.plugins, - args, - automaton, + baseline_plugins=baseline_collection.plugins, + args=args, + automaton=automaton, ) baseline_collection.plugins = plugins results = find_secrets_in_files(args, plugins) + if baseline_collection: original_results = results results = get_secrets_not_in_baseline( diff --git a/test_data/files/file_with_custom_secrets.py b/test_data/files/file_with_custom_secrets.py new file mode 100644 index 000000000..ecaee85c5 --- /dev/null +++ b/test_data/files/file_with_custom_secrets.py @@ -0,0 +1,4 @@ +#!/usr/bin/python +REGULAR_VALUE = 'this is just a long string, like a user facing error message' +FOOD = 'sweet potato casserole' +ANIMAL = 'Hippo' diff --git a/testing/custom_plugins_dir/dessert.py b/testing/custom_plugins_dir/dessert.py new file mode 100644 index 000000000..81a4e1180 --- /dev/null +++ b/testing/custom_plugins_dir/dessert.py @@ -0,0 +1,20 @@ +import re + +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.base import RegexBasedDetector + + +class DessertDetector(RegexBasedDetector): + """Scans for tasty desserts.""" + secret_type = 'Tasty Dessert' + + @classproperty + def disable_flag_text(cls): + return 'no-dessert-scan' + + denylist = ( + re.compile( + r"(reese's peanut butter chocolate cake cheesecake|sweet potato casserole)", + re.IGNORECASE, + ), + ) diff --git a/testing/hippo_plugin.py b/testing/hippo_plugin.py new file mode 100644 index 000000000..ab754471e --- /dev/null +++ b/testing/hippo_plugin.py @@ -0,0 +1,20 @@ +import re + +from detect_secrets.plugins.base import classproperty +from detect_secrets.plugins.base import RegexBasedDetector + + +class HippoDetector(RegexBasedDetector): + """Scans for hippos.""" + secret_type = 'Hippo' + + @classproperty + def disable_flag_text(cls): + return 'no-hippo-scan' + + denylist = ( + re.compile( + r'(hippo)', + re.IGNORECASE, + ), + ) diff --git a/testing/util.py b/testing/util.py index 2b08b5a28..fbddb09f4 100644 --- a/testing/util.py +++ b/testing/util.py @@ -1,5 +1,10 @@ import re +import shlex +import mock + +from detect_secrets.core.usage import ParserBuilder +from detect_secrets.main import main from detect_secrets.plugins.base import RegexBasedDetector from detect_secrets.plugins.common.util import import_plugins @@ -15,6 +20,30 @@ def uncolor(text): def get_regex_based_plugins(): return { name: plugin - for name, plugin in import_plugins().items() + for name, plugin in import_plugins(custom_plugin_paths=()).items() if issubclass(plugin, RegexBasedDetector) } + + +def parse_pre_commit_args_with_correct_prog(argument_string=''): + parser = ParserBuilder() + # Rename from pytest.py to what it is when ran + parser.parser.prog = 'detect-secrets-hook' + return parser.add_pre_commit_arguments()\ + .parse_args(argument_string.split()) + + +def wrap_detect_secrets_main(command): + with mock.patch( + 'detect_secrets.main.parse_args', + return_value=_parse_console_use_args_with_correct_prog(command), + ): + return main(command.split()) + + +def _parse_console_use_args_with_correct_prog(argument_string=''): + parser = ParserBuilder() + # Rename from pytest.py to what it is when ran + parser.parser.prog = 'detect-secrets' + return parser.add_console_use_arguments()\ + .parse_args(shlex.split(argument_string)) diff --git a/tests/core/audit_test.py b/tests/core/audit_test.py index 024554457..063388e0b 100644 --- a/tests/core/audit_test.py +++ b/tests/core/audit_test.py @@ -240,6 +240,7 @@ def mock_env(self, user_inputs=None, baseline=None): @property def baseline(self): return { + 'custom_plugin_paths': (), 'generated_at': 'some timestamp', 'plugins_used': [ { @@ -272,6 +273,7 @@ def baseline(self): @property def leapfrog_baseline(self): return { + 'custom_plugin_paths': (), 'generated_at': 'some timestamp', 'plugins_used': [ { @@ -397,6 +399,7 @@ def _get_baseline_from_file(_): @property def old_baseline(self): return { + 'custom_plugin_paths': (), 'plugins_used': [ { 'name': 'Base64HighEntropyString', @@ -441,6 +444,7 @@ def old_baseline(self): @property def new_baseline(self): return { + 'custom_plugin_paths': (), 'plugins_used': [ { 'name': 'Base64HighEntropyString', @@ -524,6 +528,7 @@ def get_audited_baseline( audited. """ baseline_fixture = { + 'custom_plugin_paths': (), 'plugins_used': plugins_used, 'results': { 'mocked_file': [ @@ -665,9 +670,11 @@ def test_determine_audit_results_secret_not_found( return_value=whole_plaintext_line, autospec=True, ): - results = audit.determine_audit_results(baseline, '.secrets.baseline') + display_results = audit.determine_audit_results(baseline, '.secrets.baseline') - hex_high_results = results['plugins']['HexHighEntropyString']['results'] + assert display_results['stats']['signal'] == '100.00%' + + hex_high_results = display_results['plugins']['HexHighEntropyString']['results'] assert len(hex_high_results['true-positives']['mocked_file']) == 1 assert hex_high_results['true-positives']['mocked_file'][0]['line'] == whole_plaintext_line assert hex_high_results['true-positives']['mocked_file'][0]['plaintext'] is None @@ -727,9 +734,9 @@ def run_logic( self, secret=None, secret_lineno=15, - settings=None, + plugins_used=None, should_find_secret=True, - force=False, + force_line_printing=False, ): # Setup default arguments if not secret: @@ -740,8 +747,8 @@ def run_logic( lineno=secret_lineno, ) - if not settings: - settings = [ + if not plugins_used: + plugins_used = [ { 'name': 'PrivateKeyDetector', }, @@ -753,12 +760,13 @@ def run_logic( should_find_secret, ): audit._print_context( - secret.filename, - secret.json(), + filename=secret.filename, + secret=secret.json(), + custom_plugin_paths=(), count=1, total=2, - plugin_settings=settings, - force=force, + plugins_used=plugins_used, + force_line_printing=force_line_printing, ) @contextmanager @@ -865,7 +873,7 @@ def test_secret_not_found_no_force(self, mock_printer): lineno=15, ), should_find_secret=False, - force=False, + force_line_printing=False, ) assert uncolor(mock_printer.message) == textwrap.dedent(""" @@ -891,7 +899,7 @@ def test_secret_not_found_force(self, mock_printer): lineno=15, ), should_find_secret=False, - force=True, + force_line_printing=True, ) assert uncolor(mock_printer.message) == textwrap.dedent(""" @@ -925,7 +933,7 @@ def test_hex_high_entropy_secret_in_yaml_file(self, mock_printer): secret='123456789a', lineno=15, ), - settings=[ + plugins_used=[ { 'name': 'HexHighEntropyString', 'hex_limit': 3, @@ -964,7 +972,7 @@ def test_keyword_secret_in_yaml_file(self, mock_printer): secret='yerba', lineno=15, ), - settings=[ + plugins_used=[ { 'name': 'KeywordDetector', }, @@ -1001,7 +1009,7 @@ def test_unicode_in_output(self, mock_printer): secret='ToCynx5Se4e2PtoZxEhW7lUJcOX15c54', lineno=10, ), - settings=[ + plugins_used=[ { 'base64_limit': 4.5, 'name': 'Base64HighEntropyString', diff --git a/tests/core/baseline_test.py b/tests/core/baseline_test.py index 360a39d4e..22b10cbff 100644 --- a/tests/core/baseline_test.py +++ b/tests/core/baseline_test.py @@ -1,4 +1,3 @@ -import json import random import mock @@ -36,6 +35,7 @@ def get_results( return baseline.initialize( path, self.plugins, + custom_plugin_paths=(), exclude_files_regex=exclude_files_regex, should_scan_all_files=scan_all_files, ).json() @@ -620,31 +620,61 @@ def get_secret(): class TestFormatBaselineForOutput: - def test_sorts_by_line_number_then_hash(self): + def test_sorts_by_line_number_then_hash_then_type(self): output_string = format_baseline_for_output({ 'results': { 'filename': [ + # Output order is reverse of this { - 'hashed_secret': 'a', + 'hashed_secret': 'f', 'line_number': 3, + 'type': 'LetterDetector', }, { - 'hashed_secret': 'z', - 'line_number': 2, + 'hashed_secret': 'a', + 'line_number': 3, + 'type': 'LetterDetector', }, { - 'hashed_secret': 'f', + 'hashed_secret': 'a', 'line_number': 3, + 'type': 'DifferentDetector', + }, + { + 'hashed_secret': 'z', + 'line_number': 2, + 'type': 'LetterDetector', }, ], }, }) - - ordered_hashes = list( - map( - lambda x: x['hashed_secret'], - json.loads(output_string)['results']['filename'], - ), + assert ''.join(output_string.split()) == ''.join( + """ + { + "results": { + "filename": [ + { + "hashed_secret": "z", + "line_number": 2, + "type": "LetterDetector" + }, + { + "hashed_secret": "a", + "line_number": 3, + "type": "DifferentDetector" + }, + { + "hashed_secret": "a", + "line_number": 3, + "type": "LetterDetector" + }, + { + "hashed_secret": "f", + "line_number": 3, + "type": "LetterDetector" + } + ] + } + } + """.split(), ) - - assert ordered_hashes == ['z', 'a', 'f'] diff --git a/tests/core/secrets_collection_test.py b/tests/core/secrets_collection_test.py index 0b7deed08..cda37dee1 100644 --- a/tests/core/secrets_collection_test.py +++ b/tests/core/secrets_collection_test.py @@ -293,7 +293,7 @@ def setup(self): def test_output(self, mock_gmtime): assert ( self.logic.format_for_baseline_output() - == self.get_point_twelve_point_seven_and_later_baseline_dict(mock_gmtime) + == self.get_point_thirteen_point_two_and_later_baseline_dict(mock_gmtime) ) def test_load_baseline_from_string_with_pre_point_twelve_string(self, mock_gmtime): @@ -317,7 +317,7 @@ def test_load_baseline_from_string_with_point_twelve_to_twelve_six_string(self, We use load_baseline_from_string as a proxy to testing load_baseline_from_dict, because it's the most entry into the private function. """ - original = self.get_point_twelve_to_twelve_six_later_baseline_dict(mock_gmtime) + original = self.get_point_twelve_and_later_baseline_dict(mock_gmtime) secrets = SecretsCollection.load_baseline_from_string( json.dumps(original), @@ -348,6 +348,10 @@ def test_load_baseline_from_string_with_point_twelve_point_seven_and_later_strin json.dumps(original), ).format_for_baseline_output() + # v0.13.2+ assertions + assert 'custom_plugin_paths' not in original + assert secrets['custom_plugin_paths'] == () + # v0.12.7+ assertions assert original['word_list']['file'] == secrets['word_list']['file'] # Original hash is thrown out and replaced with new word list hash @@ -383,15 +387,21 @@ def test_load_baseline_without_exclude(self, mock_log): ) assert mock_log.error_messages == 'Incorrectly formatted baseline!\n' + def get_point_thirteen_point_two_and_later_baseline_dict(self, gmtime): + # In v0.13.2 --custom-plugins got added + baseline = self.get_point_twelve_point_seven_and_later_baseline_dict(gmtime) + baseline['custom_plugin_paths'] = () + return baseline + def get_point_twelve_point_seven_and_later_baseline_dict(self, gmtime): # In v0.12.7 --word-list got added - baseline = self.get_point_twelve_to_twelve_six_later_baseline_dict(gmtime) + baseline = self.get_point_twelve_and_later_baseline_dict(gmtime) baseline['word_list'] = {} baseline['word_list']['file'] = 'will_be_mocked.txt' baseline['word_list']['hash'] = '5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8' return baseline - def get_point_twelve_to_twelve_six_later_baseline_dict(self, gmtime): + def get_point_twelve_and_later_baseline_dict(self, gmtime): # In v0.12.0 `exclude_regex` got replaced by `exclude` baseline = self._get_baseline_dict(gmtime) baseline['exclude'] = {} diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py index 6cf3ee40f..94a29f2f2 100644 --- a/tests/core/usage_test.py +++ b/tests/core/usage_test.py @@ -1,32 +1,25 @@ import pytest -from detect_secrets.core.usage import ParserBuilder from detect_secrets.plugins.common.util import import_plugins +from testing.util import parse_pre_commit_args_with_correct_prog class TestPluginOptions: - @staticmethod - def parse_args(argument_string=''): - # PluginOptions are added in pre-commit hook - return ParserBuilder()\ - .add_pre_commit_arguments()\ - .parse_args(argument_string.split()) - def test_added_by_default(self): # This is what happens with unrecognized arguments with pytest.raises(SystemExit): - self.parse_args('--unrecognized-argument') + parse_pre_commit_args_with_correct_prog('--unrecognized-argument') - self.parse_args('--no-private-key-scan') + parse_pre_commit_args_with_correct_prog('--no-private-key-scan') def test_consolidates_output_basic(self): """Everything enabled by default, with default values""" - args = self.parse_args() + args = parse_pre_commit_args_with_correct_prog() regex_based_plugins = { key: {} - for key in import_plugins() + for key in import_plugins(custom_plugin_paths=()) } regex_based_plugins.update({ 'HexHighEntropyString': { @@ -42,10 +35,14 @@ def test_consolidates_output_basic(self): assert not hasattr(args, 'no_private_key_scan') def test_consolidates_removes_disabled_plugins(self): - args = self.parse_args('--no-private-key-scan') + args = parse_pre_commit_args_with_correct_prog('--no-private-key-scan') assert 'PrivateKeyDetector' not in args.plugins + def test_help(self): + with pytest.raises(SystemExit): + parse_pre_commit_args_with_correct_prog('--help') + @pytest.mark.parametrize( 'argument_string,expected_value', [ @@ -59,13 +56,31 @@ def test_consolidates_removes_disabled_plugins(self): ) def test_custom_limit(self, argument_string, expected_value): if expected_value is not None: - args = self.parse_args(argument_string) + args = parse_pre_commit_args_with_correct_prog(argument_string) assert ( args.plugins['HexHighEntropyString']['hex_limit'] + == expected_value + ) + else: + with pytest.raises(SystemExit): + parse_pre_commit_args_with_correct_prog(argument_string) + @pytest.mark.parametrize( + 'argument_string,expected_value', + [ + ('--custom-plugins testing', ('testing',)), + ('--custom-plugins does_not_exist', None), + ], + ) + def test_custom_plugins(self, argument_string, expected_value): + if expected_value is not None: + args = parse_pre_commit_args_with_correct_prog(argument_string) + + assert ( + args.custom_plugin_paths == expected_value ) else: with pytest.raises(SystemExit): - self.parse_args(argument_string) + parse_pre_commit_args_with_correct_prog(argument_string) diff --git a/tests/main_test.py b/tests/main_test.py index 597b3b80a..182e1bf0b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,5 +1,4 @@ import json -import shlex import textwrap from contextlib import contextmanager @@ -9,12 +8,12 @@ from detect_secrets import main as main_module from detect_secrets import VERSION from detect_secrets.core import audit as audit_module -from detect_secrets.main import main from detect_secrets.plugins.common.util import import_plugins from testing.factories import secrets_collection_factory from testing.mocks import Any from testing.mocks import mock_printer from testing.util import uncolor +from testing.util import wrap_detect_secrets_main def get_list_of_plugins(include=None, exclude=None): @@ -31,7 +30,7 @@ def get_list_of_plugins(include=None, exclude=None): ] output = [] - for name, plugin in import_plugins().items(): + for name, plugin in import_plugins(custom_plugin_paths=()).items(): if ( name in included_plugins or exclude and name in exclude @@ -48,19 +47,19 @@ def get_list_of_plugins(include=None, exclude=None): if include: output.extend(include) - return sorted(output, key=lambda x: x['name']) + return sorted(output, key=lambda plugin: plugin['name']) def get_plugin_report(extra=None): """ :type extra: Dict[str, str] """ - if not extra: # pragma: no cover + if not extra: # pragma: no cover extra = {} longest_name_length = max([ len(name) - for name in import_plugins() + for name in import_plugins(custom_plugin_paths=()) ]) return '\n'.join( @@ -69,7 +68,7 @@ def get_plugin_report(extra=None): name=name + ' ' * (longest_name_length - len(name)), result='False' if name not in extra else extra[name], ) - for name in import_plugins() + for name in import_plugins(custom_plugin_paths=()) ]), ) + '\n' @@ -81,10 +80,11 @@ class TestMain: def test_scan_basic(self, mock_baseline_initialize): with mock_stdin(): - assert main(['scan']) == 0 + assert wrap_detect_secrets_main('scan') == 0 mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + custom_plugin_paths=Any(tuple), exclude_files_regex=None, exclude_lines_regex=None, path='.', @@ -95,10 +95,11 @@ def test_scan_basic(self, mock_baseline_initialize): def test_scan_with_rootdir(self, mock_baseline_initialize): with mock_stdin(): - assert main('scan test_data'.split()) == 0 + assert wrap_detect_secrets_main('scan test_data') == 0 mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + custom_plugin_paths=Any(tuple), exclude_files_regex=None, exclude_lines_regex=None, path=['test_data'], @@ -109,12 +110,13 @@ def test_scan_with_rootdir(self, mock_baseline_initialize): def test_scan_with_exclude_args(self, mock_baseline_initialize): with mock_stdin(): - assert main( - 'scan --exclude-files some_pattern_here --exclude-lines other_patt'.split(), + assert wrap_detect_secrets_main( + 'scan --exclude-files some_pattern_here --exclude-lines other_patt', ) == 0 mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + custom_plugin_paths=Any(tuple), exclude_files_regex='some_pattern_here', exclude_lines_regex='other_patt', path='.', @@ -155,7 +157,7 @@ def test_scan_string_basic( ), mock_printer( main_module, ) as printer_shim: - assert main('scan --string'.split()) == 0 + assert wrap_detect_secrets_main('scan --string') == 0 assert uncolor(printer_shim.message) == get_plugin_report({ 'Base64HighEntropyString': expected_base64_result, 'HexHighEntropyString': expected_hex_result, @@ -169,7 +171,7 @@ def test_scan_string_cli_overrides_stdin(self): ), mock_printer( main_module, ) as printer_shim: - assert main('scan --string 012345'.split()) == 0 + assert wrap_detect_secrets_main('scan --string 012345') == 0 assert uncolor(printer_shim.message) == get_plugin_report({ 'Base64HighEntropyString': 'False (2.585)', 'HexHighEntropyString': 'False (2.121)', @@ -177,10 +179,11 @@ def test_scan_string_cli_overrides_stdin(self): def test_scan_with_all_files_flag(self, mock_baseline_initialize): with mock_stdin(): - assert main('scan --all-files'.split()) == 0 + assert wrap_detect_secrets_main('scan --all-files') == 0 mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + custom_plugin_paths=Any(tuple), exclude_files_regex=None, exclude_lines_regex=None, path='.', @@ -191,7 +194,7 @@ def test_scan_with_all_files_flag(self, mock_baseline_initialize): def test_reads_from_stdin(self, mock_merge_baseline): with mock_stdin(json.dumps({'key': 'value'})): - assert main(['scan']) == 0 + assert wrap_detect_secrets_main('scan') == 0 mock_merge_baseline.assert_called_once_with( {'key': 'value'}, @@ -205,7 +208,7 @@ def test_reads_old_baseline_from_file(self, mock_merge_baseline): ) as m_read, mock.patch( 'detect_secrets.main.write_baseline_to_file', ) as m_write: - assert main('scan --update old_baseline_file'.split()) == 0 + assert wrap_detect_secrets_main('scan --update old_baseline_file') == 0 assert m_read.call_args[0][0] == 'old_baseline_file' assert m_write.call_args[1]['filename'] == 'old_baseline_file' assert m_write.call_args[1]['data'] == Any(dict) @@ -245,11 +248,9 @@ def test_old_baseline_ignored_with_update_flag( # We don't want to be creating a file during test 'detect_secrets.main.write_baseline_to_file', ) as file_writer: - assert main( - shlex.split( - 'scan --update old_baseline_file {}'.format( - exclude_files_arg, - ), + assert wrap_detect_secrets_main( + 'scan --update old_baseline_file {}'.format( + exclude_files_arg, ), ) == 0 @@ -435,11 +436,9 @@ def test_plugin_from_old_baseline_respected_with_update_flag( # We don't want to be creating a file during test 'detect_secrets.main.write_baseline_to_file', ) as file_writer: - assert main( - shlex.split( - 'scan --update old_baseline_file {}'.format( - plugins_overwriten, - ), + assert wrap_detect_secrets_main( + 'scan --update old_baseline_file {}'.format( + plugins_overwriten, ), ) == 0 @@ -489,10 +488,11 @@ def test_audit_short_file(self, filename, expected_output): # To extract the baseline output main_module, ) as printer_shim: - main(['scan', filename]) + wrap_detect_secrets_main('scan ' + filename) baseline = printer_shim.message baseline_dict = json.loads(baseline) + baseline_dict['custom_plugin_paths'] = tuple(baseline_dict['custom_plugin_paths']) with mock_stdin(), mock.patch( # To pipe in printer_shim 'detect_secrets.core.audit._get_baseline_from_file', @@ -510,7 +510,7 @@ def test_audit_short_file(self, filename, expected_output): ), mock_printer( audit_module, ) as printer_shim: - main('audit will_be_mocked'.split()) + wrap_detect_secrets_main('audit will_be_mocked') assert uncolor(printer_shim.message) == textwrap.dedent(""" Secret: 1 of 1 @@ -556,26 +556,27 @@ def test_audit_display_results(self, filename, expected_output): with mock_stdin(), mock_printer( main_module, ) as printer_shim: - main(['scan', filename]) + wrap_detect_secrets_main('scan ' + filename) baseline = printer_shim.message baseline_dict = json.loads(baseline) + baseline_dict['custom_plugin_paths'] = tuple(baseline_dict['custom_plugin_paths']) with mock.patch( 'detect_secrets.core.audit._get_baseline_from_file', return_value=baseline_dict, ), mock_printer( audit_module, ) as printer_shim: - main(['audit', '--display-results', 'MOCKED']) + wrap_detect_secrets_main('audit --display-results MOCKED') assert json.loads(uncolor(printer_shim.message))['plugins'] == expected_output def test_audit_diff_not_enough_files(self): - assert main('audit --diff fileA'.split()) == 1 + assert wrap_detect_secrets_main('audit --diff fileA') == 1 def test_audit_same_file(self): with mock_printer(main_module) as printer_shim: - assert main('audit --diff .secrets.baseline .secrets.baseline'.split()) == 0 + assert wrap_detect_secrets_main('audit --diff .secrets.baseline .secrets.baseline') == 0 assert printer_shim.message.strip() == ( 'No difference, because it\'s the same file!' ) diff --git a/tests/plugins/common/initialize_test.py b/tests/plugins/common/initialize_test.py index c3aa26a41..b9ef5d613 100644 --- a/tests/plugins/common/initialize_test.py +++ b/tests/plugins/common/initialize_test.py @@ -10,29 +10,37 @@ class TestFromPluginClassname: def test_success(self): plugin = initialize.from_plugin_classname( - 'HexHighEntropyString', + plugin_classname='HexHighEntropyString', + custom_plugin_paths=(), hex_limit=4, ) - assert isinstance(plugin, HexHighEntropyString) + # Dynamically imported classes have different + # addresses for the same functions as statically + # imported classes do, so isinstance does not work. + assert str(plugin.__class__) == str(HexHighEntropyString) + assert dir(plugin.__class__) == dir(HexHighEntropyString) + assert plugin.entropy_limit == 4 def test_fails_if_not_base_plugin(self): with pytest.raises(TypeError): initialize.from_plugin_classname( - 'log', + plugin_classname='NotABasePlugin', + custom_plugin_paths=(), ) def test_fails_on_bad_initialization(self): - with mock.patch.object( - HexHighEntropyString, - '__init__', - side_effect=TypeError, + with mock.patch( + 'detect_secrets.plugins.common.initialize.import_plugins', + # Trying to instantiate str() like a plugin throws TypeError + return_value={'HexHighEntropyString': str}, ), pytest.raises( TypeError, ): initialize.from_plugin_classname( - 'HexHighEntropyString', + plugin_classname='HexHighEntropyString', + custom_plugin_paths=(), hex_limit=4, ) @@ -40,7 +48,7 @@ def test_fails_on_bad_initialization(self): class TestFromSecretType: def setup(self): - self.settings = [ + self.plugins_used = [ { 'name': 'Base64HighEntropyString', 'base64_limit': 3, @@ -53,20 +61,27 @@ def setup(self): def test_success(self): plugin = initialize.from_secret_type( 'Base64 High Entropy String', - settings=self.settings, + plugins_used=self.plugins_used, + custom_plugin_paths=(), ) + # Dynamically imported classes have different + # addresses for the same functions as statically + # imported classes do, so isinstance does not work. + assert str(plugin.__class__) == str(Base64HighEntropyString) + assert dir(plugin.__class__) == dir(Base64HighEntropyString) - assert isinstance(plugin, Base64HighEntropyString) assert plugin.entropy_limit == 3 def test_failure(self): assert not initialize.from_secret_type( 'some random secret_type', - settings=self.settings, + plugins_used=self.plugins_used, + custom_plugin_paths=(), ) def test_secret_type_not_in_settings(self): assert not initialize.from_secret_type( 'Base64 High Entropy String', - settings=[], + plugins_used=[], + custom_plugin_paths=(), ) diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index aa4941d58..db92cfcf2 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -12,18 +12,27 @@ from testing.mocks import mock_log as mock_log_base from testing.mocks import SubprocessMock from testing.util import get_regex_based_plugins +from testing.util import parse_pre_commit_args_with_correct_prog + + +def call_pre_commit_hook(command): + with mock.patch( + 'detect_secrets.pre_commit_hook.parse_args', + return_value=parse_pre_commit_args_with_correct_prog(command), + ): + return pre_commit_hook.main(command.split()) def assert_commit_blocked(command): - assert pre_commit_hook.main(command.split()) == 1 + assert call_pre_commit_hook(command) == 1 def assert_commit_blocked_with_diff_exit_code(command): - assert pre_commit_hook.main(command.split()) == 3 + assert call_pre_commit_hook(command) == 3 def assert_commit_succeeds(command): - assert pre_commit_hook.main(command.split()) == 0 + assert call_pre_commit_hook(command) == 0 class TestPreCommitHook: @@ -41,19 +50,58 @@ def test_file_with_secrets(self, mock_log): assert message_by_lines[0].startswith( 'Potential secrets about to be committed to git repo!', ) - assert message_by_lines[2] == \ + assert ( + message_by_lines[2] + == 'Secret Type: Base64 High Entropy String' - assert message_by_lines[3] == \ + ) + assert ( + message_by_lines[3] + == 'Location: test_data/files/file_with_secrets.py:3' + ) def test_file_with_secrets_with_word_list(self): assert_commit_succeeds( 'test_data/files/file_with_secrets.py --word-list test_data/word_list.txt', ) - def test_file_no_secrets(self): + def test_file_with_no_secrets(self): assert_commit_succeeds('test_data/files/file_with_no_secrets.py') + def test_file_with_custom_secrets_without_custom_plugins(self): + assert_commit_succeeds('test_data/files/file_with_custom_secrets.py') + + def test_file_with_custom_secrets_with_custom_plugins(self, mock_log): + assert_commit_blocked( + 'test_data/files/file_with_custom_secrets.py ' + '--custom-plugins testing/custom_plugins_dir ' + '--custom-plugins testing/hippo_plugin.py', + ) + + message_by_lines = list( + filter( + lambda x: x != '', + mock_log.error_messages.splitlines(), + ), + ) + assert message_by_lines[0].startswith( + 'Potential secrets about to be committed to git repo!', + ) + assert 'Secret Type: Hippo' in message_by_lines + assert ( + 'Location: test_data/files/file_with_custom_secrets.py:4' + in message_by_lines + ) + assert ( + 'Secret Type: Tasty Dessert' + in message_by_lines + ) + assert ( + 'Location: test_data/files/file_with_custom_secrets.py:3' + in message_by_lines + ) + @pytest.mark.parametrize( 'has_result, use_private_key_scan, hook_command, commit_succeeds', [ diff --git a/tox.ini b/tox.ini index 880cac27f..dd23fea0c 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,8 @@ commands = coverage run -m pytest tests coverage report --show-missing --include=tests/* --fail-under 100 # This is so that we do not regress unintentionally - coverage report --show-missing --include=detect_secrets/* --omit=detect_secrets/core/audit.py,detect_secrets/core/secrets_collection.py,detect_secrets/main.py,detect_secrets/plugins/common/ini_file_parser.py --fail-under 99 - coverage report --show-missing --include=detect_secrets/core/audit.py,detect_secrets/core/secrets_collection.py,detect_secrets/main.py,detect_secrets/plugins/common/ini_file_parser.py --fail-under 96 + coverage report --show-missing --include=detect_secrets/* --omit=detect_secrets/core/audit.py,detect_secrets/core/secrets_collection.py,detect_secrets/main.py,detect_secrets/plugins/common/ini_file_parser.py,detect_secrets/plugins/cloudant.py,detect_secrets/plugins/softlayer.py --fail-under 100 + coverage report --show-missing --include=detect_secrets/core/audit.py,detect_secrets/core/secrets_collection.py,detect_secrets/main.py,detect_secrets/plugins/common/ini_file_parser.py,detect_secrets/plugins/cloudant.py,detect_secrets/plugins/softlayer.py --fail-under 96 pre-commit run --all-files --show-diff-on-failure [testenv:venv]