Multiple filters support and secret filter by regex

detect_secrets/core/usage/filters.py

@@ -31,13 +31,21 @@ def add_filter_options(parent: argparse.ArgumentParser) -> None:
     parser.add_argument(
         '--exclude-lines',
         type=str,
+        action='append',
         help='If lines match this regex, it will be ignored.',
     )
     parser.add_argument(
         '--exclude-files',
         type=str,
+        action='append',
         help='If filenames match this regex, it will be ignored.',
     )
+    parser.add_argument(
+        '--exclude-secrets',
+        type=str,
+        action='append',
+        help='If secrets match this regex, it will be ignored.',
+    )
 
     if filters.wordlist.is_feature_enabled():
         parser.add_argument(
@@ -62,6 +70,11 @@ def parse_args(args: argparse.Namespace) -> None:
             'pattern': args.exclude_files,
         }
 
+    if args.exclude_secrets:
+        get_settings().filters['detect_secrets.filters.regex.should_exclude_secret'] = {
+            'pattern': args.exclude_secrets,
+        }
+
     if (
         filters.wordlist.is_feature_enabled()
         and args.word_list_file

detect_secrets/filters/regex.py

@@ -1,28 +1,49 @@
 import re
 from functools import lru_cache
+from typing import List
 from typing import Pattern
 
 from ..settings import get_settings
 from .util import get_caller_path
 
 
 def should_exclude_line(line: str) -> bool:
-    regex = _get_line_exclusion_regex()
-    return bool(regex.search(line))
+    regexes = _get_line_exclusion_regex()
+    for regex in regexes:
+        if regex.search(line):
+            return True
+    return False
 
 
 @lru_cache(maxsize=1)
-def _get_line_exclusion_regex() -> Pattern:
+def _get_line_exclusion_regex() -> List[Pattern]:
     path = get_caller_path(offset=1)
-    return re.compile(get_settings().filters[path]['pattern'])
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]
 
 
 def should_exclude_file(filename: str) -> bool:
-    regex = _get_file_exclusion_regex()
-    return bool(regex.search(filename))
+    regexes = _get_file_exclusion_regex()
+    for regex in regexes:
+        if regex.search(filename):
+            return True
+    return False
 
 
 @lru_cache(maxsize=1)
-def _get_file_exclusion_regex() -> Pattern:
+def _get_file_exclusion_regex() -> List[Pattern]:
     path = get_caller_path(offset=1)
-    return re.compile(get_settings().filters[path]['pattern'])
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]
+
+
+def should_exclude_secret(secret: str) -> bool:
+    regexes = _get_secret_exclusion_regex()
+    for regex in regexes:
+        if regex.search(secret):
+            return True
+    return False
+
+
+@lru_cache(maxsize=1)
+def _get_secret_exclusion_regex() -> List[Pattern]:
+    path = get_caller_path(offset=1)
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]

tests/core/secrets_collection_test.py

@@ -55,7 +55,9 @@ def test_line_based_success():
             # This gets rid of the aws keys with `EXAMPLE` in them.
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_line',
-                'pattern': 'EXAMPLE',
+                'pattern': [
+                    'EXAMPLE',
+                ],
             },
         ])
 
@@ -135,7 +137,9 @@ def test_filename_filters_are_invoked_first():
         get_settings().configure_filters([
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_file',
-                'pattern': 'test|baseline',
+                'pattern': [
+                    'test|baseline',
+                ],
             },
         ])
 
@@ -335,7 +339,9 @@ def test_subtraction(configure_plugins):
         'filters_used': [
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_line',
-                'pattern': 'EXAMPLE',
+                'pattern': [
+                    'EXAMPLE',
+                ],
             },
         ],
     }):

tests/filters/regex_filter_test.py

@@ -14,9 +14,14 @@ def parser():
 
 
 def test_should_exclude_line(parser):
-    parser.parse_args(['--exclude-lines', 'canarytoken'])
+    parser.parse_args([
+        '--exclude-lines', 'canarytoken',
+        '--exclude-lines', '^not-real-secret = .*$',
+    ])
     assert filters.regex.should_exclude_line('password = "canarytoken"') is True
     assert filters.regex.should_exclude_line('password = "hunter2"') is False
+    assert filters.regex.should_exclude_line('not-real-secret = value') is True
+    assert filters.regex.should_exclude_line('maybe-not-real-secret = value') is False
 
     assert [
         item
@@ -25,15 +30,23 @@ def test_should_exclude_line(parser):
     ] == [
         {
             'path': 'detect_secrets.filters.regex.should_exclude_line',
-            'pattern': 'canarytoken',
+            'pattern': [
+                'canarytoken',
+                '^not-real-secret = .*$',
+            ],
         },
     ]
 
 
 def test_should_exclude_file(parser):
-    parser.parse_args(['--exclude-files', '^tests/.*'])
+    parser.parse_args([
+        '--exclude-files', '^tests/.*',
+        '--exclude-files', '.*/i18/.*',
+    ])
     assert filters.regex.should_exclude_file('tests/blah.py') is True
     assert filters.regex.should_exclude_file('detect_secrets/tests/blah.py') is False
+    assert filters.regex.should_exclude_file('app/messages/i18/en.properties') is True
+    assert filters.regex.should_exclude_file('app/i18secrets/secrets.yaml') is False
 
     assert [
         item
@@ -42,6 +55,34 @@ def test_should_exclude_file(parser):
     ] == [
         {
             'path': 'detect_secrets.filters.regex.should_exclude_file',
-            'pattern': '^tests/.*',
+            'pattern': [
+                '^tests/.*',
+                '.*/i18/.*',
+            ],
+        },
+    ]
+
+
+def test_should_exclude_secret(parser):
+    parser.parse_args([
+        '--exclude-secrets', '^[Pp]assword[0-9]{0,3}$',
+        '--exclude-secrets', 'my-first-password',
+    ])
+    assert filters.regex.should_exclude_secret('Password123') is True
+    assert filters.regex.should_exclude_secret('MyRealPassword') is False
+    assert filters.regex.should_exclude_secret('1-my-first-password-for-database') is True
+    assert filters.regex.should_exclude_secret('my-password') is False
+
+    assert [
+        item
+        for item in get_settings().json()['filters_used']
+        if item['path'] == 'detect_secrets.filters.regex.should_exclude_secret'
+    ] == [
+        {
+            'path': 'detect_secrets.filters.regex.should_exclude_secret',
+            'pattern': [
+                '^[Pp]assword[0-9]{0,3}$',
+                'my-first-password',
+            ],
         },
     ]