🔭 [Keyword Plugin] Accuracy improvements

- Add FP check for '/etc/' in secret - Add FP heuristic for secrets that look like directories - Add <secret> FP heuristic for .example files - Add more to FALSE_POSITIVES dict - Make all non-quotes required filetypes skip secrets starting with a $ - Make quotes required for Terraform files
Yelp · Aug 23, 2019 · a53d12e · a53d12e
1 parent 2f2ccb0
commit a53d12e
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 17 deletions.
diff --git a/detect_secrets/plugins/common/filetype.py b/detect_secrets/plugins/common/filetype.py
@@ -4,19 +4,22 @@
 
 class FileType(Enum):
     CLS = 0
-    GO = 1
-    JAVA = 2
-    JAVASCRIPT = 3
-    PHP = 4
-    OBJECTIVE_C = 5
-    PYTHON = 6
-    SWIFT = 7
-    YAML = 8
-    OTHER = 9
+    EXAMPLE = 1
+    GO = 2
+    JAVA = 3
+    JAVASCRIPT = 4
+    PHP = 5
+    OBJECTIVE_C = 6
+    PYTHON = 7
+    SWIFT = 8
+    TERRAFORM = 9
+    YAML = 10
+    OTHER = 11
 
 
 EXTENSION_TO_FILETYPE = {
     '.cls': FileType.CLS,
+    '.example': FileType.EXAMPLE,
     '.eyaml': FileType.YAML,
     '.go': FileType.GO,
     '.java': FileType.JAVA,
@@ -26,6 +29,7 @@ class FileType(Enum):
     '.py': FileType.PYTHON,
     '.pyi': FileType.PYTHON,
     '.swift': FileType.SWIFT,
+    '.tf': FileType.TERRAFORM,
     '.yaml': FileType.YAML,
     '.yml': FileType.YAML,
 }

diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py
@@ -66,8 +66,9 @@
     "'this",
     '(nsstring',
     '-default}',
-    '/etc/passwd:ro',
     '::',
+    '<%=',
+    '<?php',
     '<a',
     '<aws_secret_access_key>',
     '<input',
@@ -80,43 +81,60 @@
     "\\k.*'",
     '`cat',
     '`grep',
+    '`sudo',
     'account_password',
+    'api_key',
+    'disable',
     'dummy_secret',
     'dummy_value',
     'false',
     'false):',
     'false,',
     'false;',
+    'login_password',
     'none',
     'none,',
     'none}',
     'not',
+    'not_real_key',
     'null',
     'null,',
     'null.*"',
     "null.*'",
     'null;',
+    'pass',
+    'pass)',
     'password',
     'password)',
+    'password))',
     'password,',
     'password},',
     'prompt',
     'redacted',
+    'secret',
     'some_key',
+    'str',
     'str_to_sign',
+    'string',
+    'string)',
     'string,',
     'string;',
     'string?',
+    'string?)',
     'string}',
     'string}}',
+    'test',
     'test-access-key',
     'thisisnottherealsecret',
     'todo',
     'true',
     'true):',
     'true,',
     'true;',
+    'undef',
+    'undef,',
     '{',
+    '{{',
 }
 # Includes ], ', " as closing
 CLOSING = r'[]\'"]{0,2}'
@@ -224,6 +242,7 @@
     FileType.JAVASCRIPT,
     FileType.PYTHON,
     FileType.SWIFT,
+    FileType.TERRAFORM,
 }
 
 
@@ -294,20 +313,26 @@ def secret_generator(self, string, filetype):
 
 def probably_false_positive(lowered_secret, filetype):
     if (
-        'fake' in lowered_secret
-        or 'forgot' in lowered_secret
-        or lowered_secret in FALSE_POSITIVES
+        any(
+            false_positive in lowered_secret
+            for false_positive in (
+                '/etc/',
+                'fake',
+                'forgot',
+            )
+        ) or lowered_secret in FALSE_POSITIVES
+        or lowered_secret.count('/') >= 3
         # For e.g. "secret": "{secret}"
         or (
             lowered_secret[0] == '{'
             and lowered_secret[-1] == '}'
         ) or (
-            filetype == FileType.PHP
+            filetype not in QUOTES_REQUIRED_FILETYPES
             and lowered_secret[0] == '$'
         ) or (
-            filetype == FileType.YAML
-            and lowered_secret.startswith('{{')
-            and lowered_secret.endswith('}}')
+            filetype == FileType.EXAMPLE
+            and lowered_secret[0] == '<'
+            and lowered_secret[-1] == '>'
         )
     ):
         return True

diff --git a/tests/plugins/keyword_test.py b/tests/plugins/keyword_test.py
@@ -124,6 +124,8 @@
             'private_key "";',  # Nothing in the quotes
             'private_key \'"no spaces\';',  # Has whitespace in the secret
             'private_key "fake";',  # 'fake' in the secret
+            'private_key "some/dir/aint/a/secret";',  # 3 or more /
+            'private_key "${FOO}";',  # Starts with ${ and ends with }
             'private_key "hopenobodyfindsthisone\';',  # Double-quote does not match single-quote
             'private_key \'hopenobodyfindsthisone";',  # Single-quote does not match double-quote
         ],
@@ -360,3 +362,20 @@ def test_analyze_yaml_negatives(self, file_content, file_extension):
             'mock_filename{}'.format(file_extension),
         )
         assert len(output) == 0
+
+    @pytest.mark.parametrize(
+        'file_content',
+        STANDARD_POSITIVES,
+    )
+    def test_analyze_example_negatives(self, file_content):
+        logic = KeywordDetector()
+
+        # Make it start with `<`, (and end with `>`) so it hits our false-positive check
+        f = mock_file_object(
+            file_content.replace('m{', '<').replace('}', '>'),
+        )
+        output = logic.analyze(
+            f,
+            'mock_filename.example',
+        )
+        assert len(output) == 0