Use RegexBasedDetector.assign_regex_generator (Yelp#215)

detect_secrets/plugins/base.py

@@ -292,7 +292,7 @@ def assign_regex_generator(prefix_regex, password_keyword_regex, password_regex)
         opt_close_square_bracket = r'(?:\]|)'
         opt_dash_undrscr = r'(?:_|-|)'
         opt_space = r'(?: *)'
-        assignment = r'(?:=|:|:=|=>|::)'
+        assignment = r'(?:=|:|:=|=>| +|::)'
         return re.compile(
             r'{begin}{opt_open_square_bracket}{opt_quote}{prefix_regex}{opt_dash_undrscr}'
             '{password_keyword_regex}{opt_quote}{opt_close_square_bracket}{opt_space}'

detect_secrets/plugins/cloudant.py

@@ -61,7 +61,7 @@ class CloudantDetector(RegexBasedDetector):
 
     def verify(self, token, content, potential_secret=None):
 
-        hosts = get_host(content)
+        hosts = find_host(content)
         if not hosts:
             return VerifiedResult.UNVERIFIED
 
@@ -71,28 +71,15 @@ def verify(self, token, content, potential_secret=None):
         return VerifiedResult.VERIFIED_FALSE
 
 
-def get_host(content):
-
-    # opt means optional
-    opt_quote = r'(?:"|\'|)'
-    opt_cl = r'(?:cloudant|cl|)'
-    opt_dash_undrscr = r'(?:_|-|)'
+def find_host(content):
     opt_hostname_keyword = r'(?:hostname|host|username|id|user|userid|user-id|user-name|' \
         'name|user_id|user_name|uname)'
-    opt_space = r'(?: |)'
-    assignment = r'(?:\=|:|:=|=>)+'
     hostname = r'(\w(?:\w|_|-)+)'
-    regex = re.compile(
-        r'{opt_quote}{opt_cl}{opt_dash_undrscr}{opt_hostname_keyword}{opt_space}{opt_quote}'
-        '{assignment}{opt_space}{opt_quote}{hostname}{opt_quote}'.format(
-            opt_quote=opt_quote,
-            opt_cl=opt_cl,
-            opt_dash_undrscr=opt_dash_undrscr,
-            opt_hostname_keyword=opt_hostname_keyword,
-            opt_space=opt_space,
-            hostname=hostname,
-            assignment=assignment,
-        ), flags=re.IGNORECASE,
+
+    regex = RegexBasedDetector.assign_regex_generator(
+        prefix_regex=CloudantDetector.cl,
+        password_keyword_regex=opt_hostname_keyword,
+        password_regex=hostname,
     )
 
     return [

detect_secrets/plugins/db2.py

@@ -136,24 +136,10 @@ def verify_db2_credentials(
 
 
 def find_other_factor(content, factor_keyword_regex, factor_regex):
-    begin = r'(?:(?<=\W)|(?<=^))'
-    opt_quote = r'(?:"|\'|)'
-    opt_db = r'(?:db2|dashdb|db|)'
-    opt_dash_undrscr = r'(?:_|-|)'
-    opt_space = r'(?: *)'
-    assignment = r'(?:=|:|:=|=>|::)'
-    regex = re.compile(
-        r'{begin}{opt_quote}{opt_db}{opt_dash_undrscr}{factor_keyword}{opt_quote}{opt_space}'
-        '{assignment}{opt_space}{opt_quote}{factor}{opt_quote}'.format(
-            begin=begin,
-            opt_quote=opt_quote,
-            opt_db=opt_db,
-            opt_dash_undrscr=opt_dash_undrscr,
-            factor_keyword=factor_keyword_regex,
-            opt_space=opt_space,
-            assignment=assignment,
-            factor=factor_regex,
-        ), flags=re.IGNORECASE,
+    regex = RegexBasedDetector.assign_regex_generator(
+        prefix_regex=DB2Detector.opt_db,
+        password_keyword_regex=factor_keyword_regex,
+        password_regex=factor_regex,
     )
 
     return [

detect_secrets/plugins/gh.py

@@ -12,32 +12,20 @@ class GHDetector(RegexBasedDetector):
 
     secret_type = 'GitHub Credentials'
 
-    opt_github = r'(?:github|gh|ghe|git|)'
+    opt_github_prefix = r'(?:github|gh|ghe|git|)(?:_|-|)(?:api|)'
     opt_space = r'(?: *)'
     opt_quote = r'(?:"|\'|)'
-    opt_assignment = r'(?:=|:|:=|=>|)'
-    opt_dash_undrscr = r'(?:_|-|)'
-    opt_api = r'(?:api|)'
     header_keyword = r'(?:token|bearer|Basic)'
     key_or_pass = r'(?:key|pwd|password|pass|token|oauth)'
     api_endpoint = r'(?:github.ibm.com|api.github.ibm.com)'
     forty_hex = r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))'
     b64_encoded_token = r'(?:(?<=\W)|(?<=^))([A-Za-z0-9+/]{55}=)(?:(?=\W)|(?=$))'
     opt_username = r'(?:[a-zA-Z0-9-]+:|)'
     denylist = [
-        re.compile(
-            r'{opt_quote}{opt_github}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}{key_or_pass}'
-            '{opt_quote}{opt_space}{opt_assignment}{opt_space}{opt_quote}{forty_hex}'
-            '{opt_quote}'.format(
-                opt_quote=opt_quote,
-                opt_github=opt_github,
-                opt_dash_undrscr=opt_dash_undrscr,
-                opt_api=opt_api,
-                key_or_pass=key_or_pass,
-                opt_space=opt_space,
-                opt_assignment=opt_assignment,
-                forty_hex=forty_hex,
-            ), flags=re.IGNORECASE,
+        RegexBasedDetector.assign_regex_generator(
+            prefix_regex=opt_github_prefix,
+            password_keyword_regex=key_or_pass,
+            password_regex=forty_hex,
         ),
         re.compile(
             r'https://{opt_username}{forty_hex}@{api_endpoint}'.format(

detect_secrets/plugins/ibm_cloud_iam.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import re
-
 import requests
 
 from .base import RegexBasedDetector
@@ -13,31 +11,17 @@ class IBMCloudIAMDetector(RegexBasedDetector):
     secret_type = 'IBM Cloud IAM Key'
 
     # opt means optional
-    opt_quote = r'(?:"|\'|)'
-    opt_dashes = r'(?:--|)'
     ibm_cloud_iam = r'(?:ibm(?:_|-|)cloud(?:_|-|)iam|cloud(?:_|-|)iam|' + \
                     r'ibm(?:_|-|)cloud|ibm(?:_|-|)iam|ibm|iam|cloud)'
     opt_dash_undrscr = r'(?:_|-|)'
     opt_api = r'(?:api|)'
     key_or_pass = r'(?:key|pwd|password|pass|token)'
-    opt_space = r'(?: *)'
-    opt_assignment = r'(?:=|:|:=|=>|)'
     secret = r'([a-zA-z0-9_\-]{44})'
     denylist = [
-        re.compile(
-            r'{opt_quote}{opt_dashes}{ibm_cloud_iam}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}'
-            '{key_or_pass}{opt_quote}{opt_space}{opt_assignment}{opt_space}{opt_quote}'
-            '{secret}{opt_quote}'.format(
-                opt_quote=opt_quote,
-                opt_dashes=opt_dashes,
-                ibm_cloud_iam=ibm_cloud_iam,
-                opt_dash_undrscr=opt_dash_undrscr,
-                opt_api=opt_api,
-                key_or_pass=key_or_pass,
-                opt_space=opt_space,
-                opt_assignment=opt_assignment,
-                secret=secret,
-            ), flags=re.IGNORECASE,
+        RegexBasedDetector.assign_regex_generator(
+            prefix_regex=ibm_cloud_iam + opt_dash_undrscr + opt_api,
+            password_keyword_regex=key_or_pass,
+            password_regex=secret,
         ),
     ]
 

detect_secrets/plugins/ibm_cos_hmac.py

@@ -28,25 +28,8 @@ class IBMCosHmacDetector(RegexBasedDetector):
         ),
     )
 
-    def get_access_key_id(self, content):
-        key_id_keyword_regex = r'(?:access[-_]?(?:key)?[-_]?(?:id)?|key[-_]?id)'
-        key_id_regex = r'([a-f0-9]{32})'
-
-        regex = RegexBasedDetector.assign_regex_generator(
-            prefix_regex=self.token_prefix,
-            password_keyword_regex=key_id_keyword_regex,
-            password_regex=key_id_regex,
-        )
-
-        return [
-            match
-            for line in content.splitlines()
-            for match in regex.findall(line)
-        ]
-
     def verify(self, token, content, potential_secret=None):
-
-        key_id_matches = self.get_access_key_id(content)
+        key_id_matches = find_access_key_id(content)
 
         if not key_id_matches:
             return VerifiedResult.UNVERIFIED
@@ -65,6 +48,23 @@ def verify(self, token, content, potential_secret=None):
         return VerifiedResult.VERIFIED_FALSE
 
 
+def find_access_key_id(content):
+    key_id_keyword_regex = r'(?:access[-_]?(?:key)?[-_]?(?:id)?|key[-_]?id)'
+    key_id_regex = r'([a-f0-9]{32})'
+
+    regex = RegexBasedDetector.assign_regex_generator(
+        prefix_regex=IBMCosHmacDetector.token_prefix,
+        password_keyword_regex=key_id_keyword_regex,
+        password_regex=key_id_regex,
+    )
+
+    return [
+        match
+        for line in content.splitlines()
+        for match in regex.findall(line)
+    ]
+
+
 def hash(key, msg):
     return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
 

detect_secrets/plugins/softlayer.py

@@ -10,39 +10,24 @@ class SoftLayerDetector(RegexBasedDetector):
     secret_type = 'SoftLayer Credentials'
 
     # opt means optional
-    opt_quote = r'(?:"|\'|)'
-    opt_dashes = r'(?:--|)'
-    sl = r'(?:softlayer|sl)'
-    opt_dash_undrscr = r'(?:_|-|)'
-    opt_api = r'(?:api|)'
+    sl = r'(?:softlayer|sl)(?:_|-|)(?:api|)'
     key_or_pass = r'(?:key|pwd|password|pass|token)'
-    opt_space = r'(?: *)'
-    opt_assignment = r'(?:=|:|:=|=>|)'
     secret = r'([a-z0-9]{64})'
     denylist = [
-        re.compile(
-            r'{opt_quote}{opt_dashes}{sl}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}{key_or_pass}'
-            '{opt_quote}{opt_space}{opt_assignment}{opt_space}{opt_quote}{secret}'
-            '{opt_quote}'.format(
-                opt_quote=opt_quote,
-                opt_dashes=opt_dashes,
-                sl=sl,
-                opt_dash_undrscr=opt_dash_undrscr,
-                opt_api=opt_api,
-                key_or_pass=key_or_pass,
-                opt_space=opt_space,
-                opt_assignment=opt_assignment,
-                secret=secret,
-            ), flags=re.IGNORECASE,
+        RegexBasedDetector.assign_regex_generator(
+            prefix_regex=sl,
+            password_keyword_regex=key_or_pass,
+            password_regex=secret,
         ),
+
         re.compile(
             r'(?:http|https)://api.softlayer.com/soap/(?:v3|v3.1)/([a-z0-9]{64})',
             flags=re.IGNORECASE,
         ),
     ]
 
     def verify(self, token, content, potential_secret=None):
-        usernames = get_username(content)
+        usernames = find_username(content)
         if not usernames:
             return VerifiedResult.UNVERIFIED
 
@@ -52,30 +37,17 @@ def verify(self, token, content, potential_secret=None):
         return VerifiedResult.VERIFIED_FALSE
 
 
-def get_username(content):
+def find_username(content):
     # opt means optional
-    opt_quote = r'(?:"|\'|)'
-    opt_dashes = r'(?:--|)'
-    opt_sl = r'(?:softlayer|sl|)'
-    opt_dash_undrscr = r'(?:_|-|)'
-    opt_api = r'(?:api|)'
-    username_keyword = r'(?:username|id|user|userid|user-id|user-name|name|user_id|user_name|uname)'
-    opt_space = r'(?: |)'
-    seperator = r'(?: |=|:|:=|=>)+'
+    username_keyword = r'(?:username|id|user|userid|user-id|user-name|' + \
+        r'name|user_id|user_name|uname)'
     username = r'(\w(?:\w|_|@|\.|-)+)'
     regex = re.compile(
-        r'{opt_quote}{opt_dashes}{opt_sl}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}'
-        '{username_keyword}{opt_quote}{seperator}{opt_quote}{username}{opt_quote}'.format(
-            opt_quote=opt_quote,
-            opt_dashes=opt_dashes,
-            opt_sl=opt_sl,
-            opt_dash_undrscr=opt_dash_undrscr,
-            opt_api=opt_api,
-            username_keyword=username_keyword,
-            opt_space=opt_space,
-            username=username,
-            seperator=seperator,
-        ), flags=re.IGNORECASE,
+        RegexBasedDetector.assign_regex_generator(
+            prefix_regex=SoftLayerDetector.sl,
+            password_keyword_regex=username_keyword,
+            password_regex=username,
+        ),
     )
 
     return [