Use different regexes in KeywordDetector to improve accuracy

detect_secrets/__init__.py

@@ -1 +1 @@
-VERSION = '0.10.3'
+VERSION = '0.10.4'

detect_secrets/plugins/base.py

@@ -47,6 +47,9 @@ def secret_generator(self, string):
 
         :type string: str
         :param string: the secret to scan
+
+        :rtype: iter
+        :returns: Of all the identifiers found
         """
         raise NotImplementedError
 

detect_secrets/plugins/keyword.py

@@ -26,22 +26,55 @@
 """
 from __future__ import absolute_import
 
+import re
+
 from .base import BasePlugin
 from detect_secrets.core.potential_secret import PotentialSecret
 from detect_secrets.plugins.core.constants import WHITELIST_REGEX
 
 
+# Note: All values here should be lowercase
 BLACKLIST = (
-    # NOTE all values here should be lowercase,
-    # otherwise _secret_generator can fail to match them
-    'pass =',
+    'apikey',
+    'api_key',
+    'pass',
     'password',
     'passwd',
-    'pwd',
+    'private_key',
     'secret',
     'secrete',
     'token',
 )
+# Uses lazy quantifiers
+BLACKLIST_REGEX = re.compile(
+    # Followed by double-quotes and a semi-colon
+    # e.g. private_key "something";
+    # e.g. private_key 'something';
+    r'|'.join(
+        '{}(.*?)("|\')(\\S*?)(\'|");'.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) + '|' +
+    # Followed by a :
+    # e.g. token:
+    r'|'.join(
+        '{}:'.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) + '|' +
+    # Follwed by an = sign
+    # e.g. my_password =
+    r'|'.join(
+        '{}(.*?)='.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) +
+    # For `pwd` it has to start with pwd after whitespace, it is too common
+    '|\\spwd(.*?)=',
+)
 
 
 class KeywordDetector(BasePlugin):
@@ -68,10 +101,10 @@ def analyze_string(self, string, line_num, filename):
 
         return output
 
-    def _secret_generator(self, lowercase_string):
-        for line in BLACKLIST:
-            if line in lowercase_string:
-                yield line
-
     def secret_generator(self, string):
-        return self._secret_generator(string.lower())
+        match = BLACKLIST_REGEX.search(
+            string.lower(),
+        )
+        if not match:
+            return []
+        return [match.group()]

tests/plugins/keyword_test.py

@@ -13,23 +13,79 @@ class TestKeywordDetector(object):
         'file_content',
         [
             (
-                'login_somewhere --http-password hopenobodyfindsthisone\n'
+                'apikey "something";'
             ),
             (
-                'token = "noentropy"'
+                'token "something";'
+            ),
+            (
+                'private_key \'"something\';'  # Single-quotes not double-quotes
+            ),
+            (
+                'apikey:'
+            ),
+            (
+                'the_token:'
+            ),
+            (
+                'my_password ='
+            ),
+            (
+                'some_token_for_something ='
             ),
             (
-                'PASSWORD = "verysimple"'
+                '  pwd = foo'
+            ),
+            (
+                'private_key "something";'
+            ),
+
+            (
+                'passwordone=foo\n'
+            ),
+            (
+                'API_KEY=hopenobodyfindsthisone\n'
+            ),
+            (
+                'token = "noentropy"'
             ),
         ],
     )
-    def test_analyze(self, file_content):
+    def test_analyze_positives(self, file_content):
         logic = KeywordDetector()
 
         f = mock_file_object(file_content)
         output = logic.analyze(f, 'mock_filename')
         assert len(output) == 1
         for potential_secret in output:
             assert 'mock_filename' == potential_secret.filename
-        generated = list(logic.secret_generator(file_content))
-        assert len(generated) == len(output)
+
+    @pytest.mark.parametrize(
+        'file_content',
+        [
+            (
+                'private_key \'"no spaces\';'  # Has whitespace in-between
+            ),
+            (
+                'passwordonefoo\n'  # No = or anything
+            ),
+            (
+                'api_keyhopenobodyfindsthisone:\n'  # Has char's in between api_key and :
+            ),
+            (
+                'my_pwd ='  # Does not start with pwd
+            ),
+            (
+                'token "noentropy;'  # No 2nd double-quote
+            ),
+            (
+                'token noentropy;'  # No quotes
+            ),
+        ],
+    )
+    def test_analyze_negatives(self, file_content):
+        logic = KeywordDetector()
+
+        f = mock_file_object(file_content)
+        output = logic.analyze(f, 'mock_filename')
+        assert len(output) == 0