Use different regexes in KeywordDetector to improve accuracy

detect_secrets/plugins/keyword.py

@@ -26,22 +26,54 @@
 """
 from __future__ import absolute_import
 
+import re
+
 from .base import BasePlugin
 from detect_secrets.core.potential_secret import PotentialSecret
 from detect_secrets.plugins.core.constants import WHITELIST_REGEX
 
 
+# Note: All values here should be lowercase
 BLACKLIST = (
-    # NOTE all values here should be lowercase,
-    # otherwise _secret_generator can fail to match them
-    'pass =',
+    'apikey',
+    'api_key',
+    'pass',
     'password',
     'passwd',
-    'pwd',
+    'private_key',
     'secret',
     'secrete',
     'token',
 )
+# Uses lazy quantifiers
+BLACKLIST_REGEX = re.compile(
+    # Followed by double-quotes and a semi-colon
+    # e.g. private_key "something";
+    r'|'.join(
+        '(.*?){}(.*?)"(.*?)";'.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) + '|' +
+    # Followed by a :
+    # e.g. token:
+    r'|'.join(
+        '(.*?){}[:](.*?)'.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) + '|' +
+    # Follwed by an = sign
+    # e.g. my_password =
+    r'|'.join(
+        '(.*?){}(.*?)=(.*?)'.format(
+            value,
+        )
+        for value in BLACKLIST
+    ) +
+    # pwd has to start with pwd, it is too common
+    '|^pwd(.*?)=(.*?)',
+)
 
 
 class KeywordDetector(BasePlugin):
@@ -57,7 +89,8 @@ def analyze_string(self, string, line_num, filename):
         if WHITELIST_REGEX.search(string):
             return output
 
-        for identifier in self.secret_generator(string):
+        identifier = self.secret_generator(string)
+        if identifier:
             secret = PotentialSecret(
                 self.secret_type,
                 filename,
@@ -68,10 +101,10 @@ def analyze_string(self, string, line_num, filename):
 
         return output
 
-    def _secret_generator(self, lowercase_string):
-        for line in BLACKLIST:
-            if line in lowercase_string:
-                yield line
-
     def secret_generator(self, string):
-        return self._secret_generator(string.lower())
+        match = BLACKLIST_REGEX.search(
+            string.lower(),
+        )
+        if not match:
+            return None
+        return match.group()

tests/plugins/keyword_test.py

@@ -13,23 +13,79 @@ class TestKeywordDetector(object):
         'file_content',
         [
             (
-                'login_somewhere --http-password hopenobodyfindsthisone\n'
+                'apikey "something";'
             ),
             (
-                'token = "noentropy"'
+                'token "something";'
+            ),
+            (
+                'apikey:'
+            ),
+            (
+                'the_token:'
+            ),
+            (
+                'my_password ='
+            ),
+            (
+                'some_token_for_something ='
+            ),
+            (
+                'pwd = foo'
             ),
             (
-                'PASSWORD = "verysimple"'
+                'private_key "something";'
+            ),
+
+            (
+                'passwordone=foo\n'
+            ),
+            (
+                'API_KEY=hopenobodyfindsthisone\n'
+            ),
+            (
+                'token = "noentropy"'
             ),
         ],
     )
-    def test_analyze(self, file_content):
+    def test_analyze_positives(self, file_content):
         logic = KeywordDetector()
 
         f = mock_file_object(file_content)
         output = logic.analyze(f, 'mock_filename')
+        print('output is {}'.format(output))
+        for k in output:
+            print(str(k))
         assert len(output) == 1
         for potential_secret in output:
             assert 'mock_filename' == potential_secret.filename
-        generated = list(logic.secret_generator(file_content))
-        assert len(generated) == len(output)
+
+    @pytest.mark.parametrize(
+        'file_content',
+        [
+            (
+                'private_key \'something\';'  # Single-quotes not double-quotes
+            ),
+            (
+                'passwordonefoo\n'  # No = or anything
+            ),
+            (
+                'api_keyhopenobodyfindsthisone:\n'  # Has char's in between api_key and :
+            ),
+            (
+                'my_pwd ='  # Does not start with pwd
+            ),
+            (
+                'token "noentropy;'  # No 2nd double-quote
+            ),
+            (
+                'token noentropy;'  # No quotes
+            ),
+        ],
+    )
+    def test_analyze_negatives(self, file_content):
+        logic = KeywordDetector()
+
+        f = mock_file_object(file_content)
+        output = logic.analyze(f, 'mock_filename')
+        assert len(output) == 0