From 54e64b130b30444ed42377a8d11918a32ae57cd5 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 5 Jan 2021 10:59:43 +0100 Subject: [PATCH 1/5] Secrets filter implementation --- detect_secrets/core/usage/filters.py | 11 +++++++++++ detect_secrets/filters/regex.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/detect_secrets/core/usage/filters.py b/detect_secrets/core/usage/filters.py index d1f4e69cf..f6e893951 100644 --- a/detect_secrets/core/usage/filters.py +++ b/detect_secrets/core/usage/filters.py @@ -38,6 +38,12 @@ def add_filter_options(parent: argparse.ArgumentParser) -> None: type=str, help='If filenames match this regex, it will be ignored.', ) + parser.add_argument( + '--exclude-secrets', + type=str, + action='append', + help='If secrets match this regex, it will be ignored.', + ) if filters.wordlist.is_feature_enabled(): parser.add_argument( @@ -62,6 +68,11 @@ def parse_args(args: argparse.Namespace) -> None: 'pattern': args.exclude_files, } + if args.exclude_secrets: + get_settings().filters['detect_secrets.filters.regex.should_exclude_secret'] = { + 'pattern': args.exclude_secrets, + } + if ( filters.wordlist.is_feature_enabled() and args.word_list_file diff --git a/detect_secrets/filters/regex.py b/detect_secrets/filters/regex.py index af1601d4e..418de80dc 100644 --- a/detect_secrets/filters/regex.py +++ b/detect_secrets/filters/regex.py @@ -1,5 +1,6 @@ import re from functools import lru_cache +from typing import List from typing import Pattern from ..settings import get_settings @@ -26,3 +27,17 @@ def should_exclude_file(filename: str) -> bool: def _get_file_exclusion_regex() -> Pattern: path = get_caller_path(offset=1) return re.compile(get_settings().filters[path]['pattern']) + + +def should_exclude_secret(secret: str) -> bool: + regexes = _get_secret_exclusion_regex() + for regex in regexes: + if regex.search(secret): + return True + return False + + +@lru_cache(maxsize=1) +def _get_secret_exclusion_regex() -> List[Pattern]: + path = get_caller_path(offset=1) + return [re.compile(regex) for regex in get_settings().filters[path]['pattern']] From 364bf9b48e37811d1ebfa596ba8b6c2028fb5278 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 5 Jan 2021 13:10:40 +0100 Subject: [PATCH 2/5] Secrets filter test --- tests/filters/regex_filter_test.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/filters/regex_filter_test.py b/tests/filters/regex_filter_test.py index f75a014bc..93dad0278 100644 --- a/tests/filters/regex_filter_test.py +++ b/tests/filters/regex_filter_test.py @@ -45,3 +45,28 @@ def test_should_exclude_file(parser): 'pattern': '^tests/.*', }, ] + + +def test_should_exclude_secret(parser): + parser.parse_args([ + '--exclude-secrets', '^[Pp]assword[0-9]{0,3}$', + '--exclude-secrets', 'my-first-password', + ]) + assert filters.regex.should_exclude_secret('Password123') is True + assert filters.regex.should_exclude_secret('MyRealPassword') is False + assert filters.regex.should_exclude_secret('1-my-first-password-for-database') is True + assert filters.regex.should_exclude_secret('my-password') is False + + assert [ + item + for item in get_settings().json()['filters_used'] + if item['path'] == 'detect_secrets.filters.regex.should_exclude_secret' + ] == [ + { + 'path': 'detect_secrets.filters.regex.should_exclude_secret', + 'pattern': [ + '^[Pp]assword[0-9]{0,3}$', + 'my-first-password', + ], + }, + ] From d4c468ea9149e4cc47bb0df86f417e3db47223f9 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 5 Jan 2021 13:36:58 +0100 Subject: [PATCH 3/5] Optimization of file and line filters: Multiple filters are allowed --- detect_secrets/core/usage/filters.py | 2 ++ detect_secrets/filters/regex.py | 22 ++++++++++++++-------- tests/core/secrets_collection_test.py | 12 +++++++++--- tests/filters/regex_filter_test.py | 24 ++++++++++++++++++++---- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/detect_secrets/core/usage/filters.py b/detect_secrets/core/usage/filters.py index f6e893951..ef545c9d5 100644 --- a/detect_secrets/core/usage/filters.py +++ b/detect_secrets/core/usage/filters.py @@ -31,11 +31,13 @@ def add_filter_options(parent: argparse.ArgumentParser) -> None: parser.add_argument( '--exclude-lines', type=str, + action='append', help='If lines match this regex, it will be ignored.', ) parser.add_argument( '--exclude-files', type=str, + action='append', help='If filenames match this regex, it will be ignored.', ) parser.add_argument( diff --git a/detect_secrets/filters/regex.py b/detect_secrets/filters/regex.py index 418de80dc..e0fd11a2d 100644 --- a/detect_secrets/filters/regex.py +++ b/detect_secrets/filters/regex.py @@ -8,25 +8,31 @@ def should_exclude_line(line: str) -> bool: - regex = _get_line_exclusion_regex() - return bool(regex.search(line)) + regexes = _get_line_exclusion_regex() + for regex in regexes: + if regex.search(line): + return True + return False @lru_cache(maxsize=1) -def _get_line_exclusion_regex() -> Pattern: +def _get_line_exclusion_regex() -> List[Pattern]: path = get_caller_path(offset=1) - return re.compile(get_settings().filters[path]['pattern']) + return [re.compile(regex) for regex in get_settings().filters[path]['pattern']] def should_exclude_file(filename: str) -> bool: - regex = _get_file_exclusion_regex() - return bool(regex.search(filename)) + regexes = _get_file_exclusion_regex() + for regex in regexes: + if regex.search(filename): + return True + return False @lru_cache(maxsize=1) -def _get_file_exclusion_regex() -> Pattern: +def _get_file_exclusion_regex() -> List[Pattern]: path = get_caller_path(offset=1) - return re.compile(get_settings().filters[path]['pattern']) + return [re.compile(regex) for regex in get_settings().filters[path]['pattern']] def should_exclude_secret(secret: str) -> bool: diff --git a/tests/core/secrets_collection_test.py b/tests/core/secrets_collection_test.py index f714fc697..3a47c2da2 100644 --- a/tests/core/secrets_collection_test.py +++ b/tests/core/secrets_collection_test.py @@ -55,7 +55,9 @@ def test_line_based_success(): # This gets rid of the aws keys with `EXAMPLE` in them. { 'path': 'detect_secrets.filters.regex.should_exclude_line', - 'pattern': 'EXAMPLE', + 'pattern': [ + 'EXAMPLE', + ], }, ]) @@ -135,7 +137,9 @@ def test_filename_filters_are_invoked_first(): get_settings().configure_filters([ { 'path': 'detect_secrets.filters.regex.should_exclude_file', - 'pattern': 'test|baseline', + 'pattern': [ + 'test|baseline', + ], }, ]) @@ -335,7 +339,9 @@ def test_subtraction(configure_plugins): 'filters_used': [ { 'path': 'detect_secrets.filters.regex.should_exclude_line', - 'pattern': 'EXAMPLE', + 'pattern': [ + 'EXAMPLE', + ], }, ], }): diff --git a/tests/filters/regex_filter_test.py b/tests/filters/regex_filter_test.py index 93dad0278..80f27ed52 100644 --- a/tests/filters/regex_filter_test.py +++ b/tests/filters/regex_filter_test.py @@ -14,9 +14,14 @@ def parser(): def test_should_exclude_line(parser): - parser.parse_args(['--exclude-lines', 'canarytoken']) + parser.parse_args([ + '--exclude-lines', 'canarytoken', + '--exclude-lines', '^not-real-secret = .*$', + ]) assert filters.regex.should_exclude_line('password = "canarytoken"') is True assert filters.regex.should_exclude_line('password = "hunter2"') is False + assert filters.regex.should_exclude_line('not-real-secret = value') is True + assert filters.regex.should_exclude_line('maybe-not-real-secret = value') is False assert [ item @@ -25,15 +30,23 @@ def test_should_exclude_line(parser): ] == [ { 'path': 'detect_secrets.filters.regex.should_exclude_line', - 'pattern': 'canarytoken', + 'pattern': [ + 'canarytoken', + '^not-real-secret = .*$', + ], }, ] def test_should_exclude_file(parser): - parser.parse_args(['--exclude-files', '^tests/.*']) + parser.parse_args([ + '--exclude-files', '^tests/.*', + '--exclude-files', '.*/i18/.*', + ]) assert filters.regex.should_exclude_file('tests/blah.py') is True assert filters.regex.should_exclude_file('detect_secrets/tests/blah.py') is False + assert filters.regex.should_exclude_file('app/messages/i18/en.properties') is True + assert filters.regex.should_exclude_file('app/i18secrets/secrets.yaml') is False assert [ item @@ -42,7 +55,10 @@ def test_should_exclude_file(parser): ] == [ { 'path': 'detect_secrets.filters.regex.should_exclude_file', - 'pattern': '^tests/.*', + 'pattern': [ + '^tests/.*', + '.*/i18/.*', + ], }, ] From cbb42d5dd4d766779054d801fe98341b445a613b Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 1 Feb 2021 11:05:27 +0100 Subject: [PATCH 4/5] Filters documentation --- README.md | 31 +++++++++++++++++++++++++++++++ docs/filters.md | 1 + 2 files changed, 32 insertions(+) diff --git a/README.md b/README.md index 4d8caf74b..10a1d6f1d 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,8 @@ filter options: If lines match this regex, it will be ignored. --exclude-files EXCLUDE_FILES If filenames match this regex, it will be ignored. + --exclude-secrets EXCLUDE_SECRETS + If secrets match this regex, it will be ignored. --word-list WORD_LIST_FILE Text file with a list of words, if a secret contains a word in the list we ignore it. @@ -318,6 +320,8 @@ filter options: If lines match this regex, it will be ignored. --exclude-files EXCLUDE_FILES If filenames match this regex, it will be ignored. + --exclude-secrets EXCLUDE_SECRETS + If secrets match this regex, it will be ignored. --word-list WORD_LIST_FILE Text file with a list of words, if a secret contains a word in the list we ignore it. @@ -430,6 +434,12 @@ specific pattern. You can specify a regex rule as such: $ detect-secrets scan --exclude-lines 'password = (blah|fake)' ``` +Or you can specify multiple regex rules as such: + +```bash +$ detect-secrets scan --exclude-lines 'password = blah' --exclude-lines 'password = fake' +``` + #### --exclude-files Sometimes, you want to be able to ignore certain files in your scan. You can specify a regex @@ -439,6 +449,27 @@ pattern to do so, and if the filename meets this regex pattern, it will not be s $ detect-secrets scan --exclude-files '.*\.signature$' ``` +Or you can specify multiple regex patterns as such: + +```bash +$ detect-secrets scan --exclude-files '.*\.signature$' --exclude-files '.*/i18n/.*' +``` + +#### --exclude-secrets + +Sometimes, you want to be able to ignore certain secret values in your scan. You can specify +a regex rule as such: + +```bash +$ detect-secrets scan --exclude-secrets '(fakesecret|\${.*})' +``` + +Or you can specify multiple regex rules as such: + +```bash +$ detect-secrets scan --exclude-secrets 'fakesecret' --exclude-secrets '\${.*})' +``` + #### --word-list If you know there are certain fake password values that you want to ignore, you can also use diff --git a/docs/filters.md b/docs/filters.md index 69a5087ad..ef49db14d 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -52,6 +52,7 @@ the `detect_secrets.filters` namespace. | `heuristic.is_non_text_file` | Ignores non-text files (e.g. archives, images). | | `regex.should_exclude_line` | Powers the [`--exclude-lines` functionality](../README.md#--exclude-lines). | | `regex.should_exclude_file` | Powers the [`--exclude-files` functionality](../README.md#--exclude-files). | +| `regex.should_exclude_secret` | Powers the [`--exclude-secrets` functionality](../README.md#--exclude-secrets). | | `wordlist.should_exclude_secret` | Powers the [`--word-list` functionality](../README.md#--word-list). | ## Configuring Filters From 03ad10df1eb946c1b097c6c64ea70925b4bc1fb5 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 1 Feb 2021 11:16:28 +0100 Subject: [PATCH 5/5] Upgrade script compatibility --- detect_secrets/core/upgrades/v1_0.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/detect_secrets/core/upgrades/v1_0.py b/detect_secrets/core/upgrades/v1_0.py index 24bd6580c..7db62b504 100644 --- a/detect_secrets/core/upgrades/v1_0.py +++ b/detect_secrets/core/upgrades/v1_0.py @@ -58,13 +58,17 @@ def _migrate_filters(baseline: Dict[str, Any]) -> None: if baseline['exclude'].get('files'): baseline['filters_used'].append({ 'path': 'detect_secrets.filters.regex.should_exclude_file', - 'pattern': baseline['exclude']['files'], + 'pattern': [ + baseline['exclude']['files'], + ], }) if baseline['exclude'].get('lines'): baseline['filters_used'].append({ 'path': 'detect_secrets.filters.regex.should_exclude_line', - 'pattern': baseline['exclude']['lines'], + 'pattern': [ + baseline['exclude']['lines'], + ], }) baseline.pop('exclude')