From 8a000072303f4a933f58d872279ca5dd8d7f63b9 Mon Sep 17 00:00:00 2001 From: XIANJUN ZHU Date: Fri, 11 Oct 2019 14:13:55 -0400 Subject: [PATCH] feat: enhance cloudant (#219) * feat: enhance cloudant * address comment * fix broken buid --- detect_secrets/plugins/cloudant.py | 39 +++++++++------ tests/plugins/cloudant_test.py | 78 ++++++++++++++++++------------ 2 files changed, 70 insertions(+), 47 deletions(-) diff --git a/detect_secrets/plugins/cloudant.py b/detect_secrets/plugins/cloudant.py index 2525ab995..438e5bcb1 100644 --- a/detect_secrets/plugins/cloudant.py +++ b/detect_secrets/plugins/cloudant.py @@ -13,17 +13,11 @@ class CloudantDetector(RegexBasedDetector): secret_type = 'Cloudant Credentials' # opt means optional - opt_quote = r'(?:"|\'|)' - opt_dashes = r'(?:--|)' - opt_dot = r'(?:\.|)' dot = r'\.' - cl_account = r'[0-9a-z\-\_]+' + cl_account = r'[\w\-]+' cl = r'(?:cloudant|cl|clou)' - opt_dash_undrscr = r'(?:_|-|)' opt_api = r'(?:api|)' cl_key_or_pass = opt_api + r'(?:key|pwd|pw|password|pass|token)' - opt_space = r'(?: |)' - assignment = r'(?:=|:|:=|=>)' cl_pw = r'([0-9a-f]{64})' cl_api_key = r'([a-z]{24})' colon = r'\:' @@ -69,7 +63,7 @@ class CloudantDetector(RegexBasedDetector): def verify(self, token, content, potential_secret=None): - hosts = find_host(content) + hosts = find_account(content) if not hosts: return VerifiedResult.UNVERIFIED @@ -79,20 +73,35 @@ def verify(self, token, content, potential_secret=None): return VerifiedResult.VERIFIED_FALSE -def find_host(content): +def find_account(content): opt_hostname_keyword = r'(?:hostname|host|username|id|user|userid|user-id|user-name|' \ - 'name|user_id|user_name|uname)' - hostname = r'(\w(?:\w|_|-)+)' + 'name|user_id|user_name|uname|account)' + account = r'(\w[\w\-]*)' + opt_basic_auth = r'(?:[\w\-:%]*\@)?' - regex = RegexBasedDetector.assign_regex_generator( - prefix_regex=CloudantDetector.cl, - password_keyword_regex=opt_hostname_keyword, - password_regex=hostname, + regexes = ( + RegexBasedDetector.assign_regex_generator( + prefix_regex=CloudantDetector.cl, + password_keyword_regex=opt_hostname_keyword, + password_regex=account, + ), + re.compile( + r'{http}{opt_basic_auth}{cl_account}{dot}{cloudant_api_url}'.format( + http=CloudantDetector.http, + opt_basic_auth=opt_basic_auth, + cl_account=account, + cl_api_key=CloudantDetector.cl_api_key, + dot=CloudantDetector.dot, + cloudant_api_url=CloudantDetector.cloudant_api_url, + ), + flags=re.IGNORECASE, + ), ) return [ match for line in content.splitlines() + for regex in regexes for match in regex.findall(line) ] diff --git a/tests/plugins/cloudant_test.py b/tests/plugins/cloudant_test.py index 0b88c1584..ffd0dec12 100644 --- a/tests/plugins/cloudant_test.py +++ b/tests/plugins/cloudant_test.py @@ -8,9 +8,9 @@ from detect_secrets.core.constants import VerifiedResult from detect_secrets.core.potential_secret import PotentialSecret from detect_secrets.plugins.cloudant import CloudantDetector -from detect_secrets.plugins.cloudant import find_host +from detect_secrets.plugins.cloudant import find_account -CL_HOST = 'testy_test' # also called user +CL_ACCOUNT = 'testy_-test' # also called user # only detecting 64 hex CL generated password CL_PW = 'abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234' @@ -24,33 +24,33 @@ class TestCloudantDetector(object): 'payload, should_flag', [ ( - 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com"'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com"'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ), True, ), ( - 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com/_api/v2/'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com/_api/v2/'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ), True, ), ( - 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com/_api/v2/'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com/_api/v2/'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ), True, ), ( - 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ), True, ), ( - 'https://{cl_host}:{cl_api_key}@{cl_host}.cloudant.com'.format( - cl_host=CL_HOST, cl_api_key=CL_API_KEY, + 'https://{cl_account}:{cl_api_key}@{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_api_key=CL_API_KEY, ), True, ), ( - 'https://{cl_host}:{cl_pw}.cloudant.com'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + 'https://{cl_account}:{cl_pw}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ), False, ), ('cloudant_password=\'{cl_pw}\''.format(cl_pw=CL_PW), True), @@ -70,8 +70,8 @@ def test_analyze_string(self, payload, should_flag): @responses.activate def test_verify_invalid_secret(self): - cl_api_url = 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + cl_api_url = 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ) responses.add( responses.GET, cl_api_url, @@ -80,13 +80,13 @@ def test_verify_invalid_secret(self): assert CloudantDetector().verify( CL_PW, - 'cloudant_host={}'.format(CL_HOST), + 'cloudant_host={}'.format(CL_ACCOUNT), ) == VerifiedResult.VERIFIED_FALSE @responses.activate def test_verify_valid_secret(self): - cl_api_url = 'https://{cl_host}:{cl_pw}@{cl_host}.cloudant.com'.format( - cl_host=CL_HOST, cl_pw=CL_PW, + cl_api_url = 'https://{cl_account}:{cl_pw}@{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_pw=CL_PW, ) responses.add( responses.GET, cl_api_url, @@ -95,22 +95,22 @@ def test_verify_valid_secret(self): potential_secret = PotentialSecret('test cloudant', 'test filename', CL_PW) assert CloudantDetector().verify( CL_PW, - 'cloudant_host={}'.format(CL_HOST), + 'cloudant_host={}'.format(CL_ACCOUNT), potential_secret, ) == VerifiedResult.VERIFIED_TRUE - assert potential_secret.other_factors['hostname'] == CL_HOST + assert potential_secret.other_factors['hostname'] == CL_ACCOUNT @responses.activate def test_verify_unverified_secret(self): assert CloudantDetector().verify( CL_PW, - 'cloudant_host={}'.format(CL_HOST), + 'cloudant_host={}'.format(CL_ACCOUNT), ) == VerifiedResult.UNVERIFIED def test_verify_no_secret(self): assert CloudantDetector().verify( CL_PW, - 'no_un={}'.format(CL_HOST), + 'no_un={}'.format(CL_ACCOUNT), ) == VerifiedResult.UNVERIFIED @pytest.mark.parametrize( @@ -120,19 +120,19 @@ def test_verify_no_secret(self): textwrap.dedent(""" --cloudant-hostname = {} """)[1:-1].format( - CL_HOST, + CL_ACCOUNT, ), - [CL_HOST], + [CL_ACCOUNT], ), # With quotes ( textwrap.dedent(""" - cl_host = "{}" + cl_account = "{}" """)[1:-1].format( - CL_HOST, + CL_ACCOUNT, ), - [CL_HOST], + [CL_ACCOUNT], ), # multiple candidates @@ -143,19 +143,33 @@ def test_verify_no_secret(self): CLOUDANT_USERID = '{}' cloudant-uname: {} """)[1:-1].format( - CL_HOST, + CL_ACCOUNT, 'test2_testy_test', 'test3-testy-testy', 'notanemail', ), [ - CL_HOST, + CL_ACCOUNT, 'test2_testy_test', 'test3-testy-testy', 'notanemail', ], ), + + # In URL + ( + 'https://{cl_account}:{cl_api_key}@{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, cl_api_key=CL_API_KEY, + ), + [CL_ACCOUNT], + ), + ( + 'https://{cl_account}.cloudant.com'.format( + cl_account=CL_ACCOUNT, + ), + [CL_ACCOUNT], + ), ), ) - def test_find_host(self, content, expected_output): - assert find_host(content) == expected_output + def test_find_account(self, content, expected_output): + assert find_account(content) == expected_output