🎉 Add high entropy exclude options

This is so you can specifically ignore lines with false-positives
that often trigger the high entropy plugins, e.g.
```
"AWS": [
    "CanonicalUser: 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"
]
```
can be ignored with e.g.
```
detect-secrets scan test.py --hex-high-entropy-exclude "(CanonicalUser)"
```

detect_secrets/core/usage.py

@@ -219,6 +219,7 @@ class PluginOptions(object):
             disable_help_text='Disables scanning for hex high entropy strings',
             related_args=[
                 ('--hex-limit', 3,),
+                ('--hex-high-entropy-exclude', None,),
             ],
         ),
         PluginDescriptor(
@@ -227,6 +228,7 @@ class PluginOptions(object):
             disable_help_text='Disables scanning for base64 high entropy strings',
             related_args=[
                 ('--base64-limit', 4.5,),
+                ('--base64-high-entropy-exclude', None,),
             ],
         ),
         PluginDescriptor(
@@ -264,6 +266,7 @@ def __init__(self, parser):
     def add_arguments(self):
         self._add_custom_limits()
         self._add_opt_out_options()
+        self._add_high_entropy_excludes()
 
         return self
 
@@ -340,7 +343,18 @@ def _add_custom_limits(self):
             nargs='?',
             help=high_entropy_help_text + 'defaults to 3.0.',
         )
-        return self
+
+    def _add_high_entropy_excludes(self):
+        self.parser.add_argument(
+            '--base64-high-entropy-exclude',
+            type=str,
+            help='Pass in regex to exclude false positives found by base 64 high-entropy detector.',
+        )
+        self.parser.add_argument(
+            '--hex-high-entropy-exclude',
+            type=str,
+            help='Pass in regex to exclude false positives found by hex high-entropy detector.',
+        )
 
     def _add_opt_out_options(self):
         for plugin in self.all_plugins:
@@ -351,8 +365,6 @@ def _add_opt_out_options(self):
                 default=False,
             )
 
-        return self
-
     def _argparse_minmax_type(self, string):
         """Custom type for argparse to enforce value limits"""
         value = float(string)

detect_secrets/plugins/high_entropy_strings.py

@@ -18,12 +18,12 @@
 
 IGNORED_SEQUENTIAL_STRINGS = (
     (
-        string.ascii_uppercase +
-        string.ascii_uppercase +
-        string.digits +
-        string.ascii_uppercase +
-        string.ascii_uppercase +
-        '+/'
+        string.ascii_uppercase
+        + string.ascii_uppercase
+        + string.digits
+        + string.ascii_uppercase
+        + string.ascii_uppercase
+        + '+/'
     ),
     string.hexdigits.upper() + string.hexdigits.upper(),
     string.ascii_uppercase + '=/',
@@ -39,7 +39,7 @@ class HighEntropyStringsPlugin(BasePlugin):
 
     __metaclass__ = ABCMeta
 
-    def __init__(self, charset, limit, *args):
+    def __init__(self, charset, limit, high_entropy_exclude, *args):
         if limit < 0 or limit > 8:
             raise ValueError(
                 'The limit set for HighEntropyStrings must be between 0.0 and 8.0',
@@ -49,6 +49,13 @@ def __init__(self, charset, limit, *args):
         self.entropy_limit = limit
         self.regex = re.compile(r'([\'"])([%s]+)(\1)' % charset)
 
+        self.high_entropy_exclude = None
+        if high_entropy_exclude:
+            self.high_entropy_exclude = re.compile(
+                high_entropy_exclude,
+                re.IGNORECASE,
+            )
+
     def analyze(self, file, filename):
         file_type_analyzers = (
             (self._analyze_ini_file(), configparser.Error,),
@@ -101,6 +108,12 @@ def analyze_string(self, string, line_num, filename):
         """
         output = {}
 
+        if (
+            self.high_entropy_exclude
+            and self.high_entropy_exclude.search(string)
+        ):
+            return output
+
         for result in self.secret_generator(string):
             if self.is_sequential_string(result):
                 continue
@@ -227,10 +240,11 @@ class HexHighEntropyString(HighEntropyStringsPlugin):
 
     secret_type = 'Hex High Entropy String'
 
-    def __init__(self, hex_limit, **kwargs):
+    def __init__(self, hex_limit, hex_high_entropy_exclude=None, **kwargs):
         super(HexHighEntropyString, self).__init__(
-            string.hexdigits,
-            hex_limit,
+            charset=string.hexdigits,
+            limit=hex_limit,
+            high_entropy_exclude=hex_high_entropy_exclude,
         )
 
     @property
@@ -278,10 +292,11 @@ class Base64HighEntropyString(HighEntropyStringsPlugin):
 
     secret_type = 'Base64 High Entropy String'
 
-    def __init__(self, base64_limit, **kwargs):
+    def __init__(self, base64_limit, base64_high_entropy_exclude=None, **kwargs):
         super(Base64HighEntropyString, self).__init__(
-            string.ascii_letters + string.digits + '+/=',
-            base64_limit,
+            charset=string.ascii_letters + string.digits + '+/=',
+            limit=base64_limit,
+            high_entropy_exclude=base64_high_entropy_exclude,
         )
 
     @property

tests/core/usage_test.py

@@ -27,10 +27,12 @@ def test_consolidates_output_basic(self):
         assert args.plugins == {
             'HexHighEntropyString': {
                 'hex_limit': 3,
+                'hex_high_entropy_exclude': None,
             },
             'BasicAuthDetector': {},
             'Base64HighEntropyString': {
                 'base64_limit': 4.5,
+                'base64_high_entropy_exclude': None,
             },
             'KeywordDetector': {},
             'PrivateKeyDetector': {},

tests/plugins/high_entropy_strings_test.py

@@ -11,6 +11,9 @@
 from testing.mocks import mock_file_object
 
 
+HIGH_ENTROPY_EXCLUDE = '(CanonicalUser)'
+
+
 class HighEntropyStringsTest(object):
     """
     Some explaining should be done regarding the "enforced" format of the parametrized
@@ -99,6 +102,8 @@ def test_analyze_multiple_strings_same_line(self, content_to_format, expected_re
             "'{secret}'  /* pragma: whitelist secret */",
             "'{secret}'  ' pragma: whitelist secret",
             "'{secret}'  -- pragma: whitelist secret",
+            # Test high entropy exclude regex
+            '"CanonicalUser": "{secret}"',
             # Not a string
             "{secret}",
         ],
@@ -125,7 +130,10 @@ class TestBase64HighEntropyStrings(HighEntropyStringsTest):
     def setup(self):
         super(TestBase64HighEntropyStrings, self).setup(
             # Testing default limit, as suggested by truffleHog.
-            Base64HighEntropyString(4.5),
+            Base64HighEntropyString(
+                base64_limit=4.5,
+                base64_high_entropy_exclude=HIGH_ENTROPY_EXCLUDE,
+            ),
             'c3VwZXIgc2VjcmV0IHZhbHVl',     # too short for high entropy
             'c3VwZXIgbG9uZyBzdHJpbmcgc2hvdWxkIGNhdXNlIGVub3VnaCBlbnRyb3B5',
         )
@@ -196,15 +204,19 @@ class TestHexHighEntropyStrings(HighEntropyStringsTest):
     def setup(self):
         super(TestHexHighEntropyStrings, self).setup(
             # Testing default limit, as suggested by truffleHog.
-            HexHighEntropyString(3),
+            HexHighEntropyString(
+                hex_limit=3,
+                hex_high_entropy_exclude=HIGH_ENTROPY_EXCLUDE,
+            ),
             'aaaaaa',
             '2b00042f7481c7b056c4b410d28f33cf',
         )
 
     def test_discounts_when_all_numbers(self):
         original_scanner = HighEntropyStringsPlugin(
-            string.hexdigits,
-            3,
+            charset=string.hexdigits,
+            limit=3,
+            high_entropy_exclude=HIGH_ENTROPY_EXCLUDE,
         )
 
         # This makes sure discounting works.