Merge pull request #64 from Yelp/62_reduce_sequential_false_positives

Reduce sequential string false positives
Yelp · Jul 31, 2018 · 8c2d022 · 8c2d022
2 parents 11b8768 + b5135fc
commit 8c2d022
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 8 deletions.
diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py
@@ -282,20 +282,20 @@ def consolidate_args(args):
     def _add_custom_limits(self):
         high_entropy_help_text = (
             'Sets the entropy limit for high entropy strings. '
-            'Value must be between 0.0 and 8.0.'
+            'Value must be between 0.0 and 8.0, '
         )
 
         self.parser.add_argument(
             '--base64-limit',
             type=self._argparse_minmax_type,
             nargs='?',
-            help=high_entropy_help_text,
+            help=high_entropy_help_text + 'defaults to 4.5.',
         )
         self.parser.add_argument(
             '--hex-limit',
             type=self._argparse_minmax_type,
             nargs='?',
-            help=high_entropy_help_text,
+            help=high_entropy_help_text + 'defaults to 3.0.',
         )
         return self
 

diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py
@@ -17,6 +17,18 @@
 from detect_secrets.plugins.core.yaml_file_parser import YamlFileParser
 
 
+IGNORED_SEQUENTIAL_STRINGS = (
+    (
+        string.ascii_uppercase +
+        string.ascii_uppercase +
+        string.digits +
+        string.ascii_uppercase +
+        string.ascii_uppercase +
+        '+/'
+    ),
+    string.hexdigits.upper() + string.hexdigits.upper(),
+    string.ascii_uppercase + '=/',
+)
 YAML_EXTENSIONS = (
     '.yaml',
     '.yml',
@@ -75,12 +87,16 @@ def analyze_string(self, string, line_num, filename):
         """Searches string for custom pattern, and captures all high entropy strings that
         match self.regex, with a limit defined as self.entropy_limit.
         """
-
         output = {}
 
         if WHITELIST_REGEX.search(string):
             return output
 
+        uppercased_string = string.upper()
+        for sequential_string in IGNORED_SEQUENTIAL_STRINGS:
+            if uppercased_string in sequential_string:
+                return output
+
         for result in self.secret_generator(string):
             secret = PotentialSecret(self.secret_type, filename, line_num, result)
             output[secret] = secret

diff --git a/test_data/config.yaml b/test_data/config.yaml
@@ -2,6 +2,7 @@ credentials:
     some_value_here: not_a_secret
     other_value_here: 1234567890a
     nested:
+        value: AKIAabcdefghijklmnop
         value: abcdefghijklmnop
 list_of_keys:
     - 123

diff --git a/test_data/short_files/last_line.ini b/test_data/short_files/last_line.ini
@@ -2,4 +2,4 @@
 secrets_for_no_one_to_find =
     hunter2
     password123
-    0123456789a
+    BEEF0123456789a
diff --git a/tests/main_test.py b/tests/main_test.py
@@ -172,7 +172,7 @@ def test_old_baseline_ignored_with_update_flag(
                     2:secrets_for_no_one_to_find =
                     3:    hunter2
                     4:    password123
-                    5:    0123456789a
+                    5:    BEEF0123456789a
                 """)[1:-1],
             ),
         ],

diff --git a/tests/plugins/high_entropy_strings_test.py b/tests/plugins/high_entropy_strings_test.py
@@ -148,12 +148,11 @@ def test_yaml_file(self):
         with open('test_data/config.yaml') as f:
             secrets = plugin.analyze(f, 'test_data/config.yaml')
 
-        assert len(secrets.values()) == 2
+        assert len(secrets.values()) == 1
         for secret in secrets.values():
             location = str(secret).splitlines()[1]
             assert location in (
                 'Location:    test_data/config.yaml:3',
-                'Location:    test_data/config.yaml:5',
             )
 
     def test_entropy_lower_limit(self):