From 753d0010f1ed0f6d13d961a0b293a889f16a62ce Mon Sep 17 00:00:00 2001 From: "Justin D. Eyster" Date: Thu, 5 Sep 2019 16:09:21 -0400 Subject: [PATCH] Delete GHDetector V1 (#186) * Delete GHDetector V1 Follow up of #184 * Move verify functionality to GHv2 * Addressed @xianjun comment * Address @xianjun comments --- detect_secrets/core/usage.py | 6 -- detect_secrets/plugins/common/initialize.py | 1 - detect_secrets/plugins/gh.py | 53 +++++++++++++++- detect_secrets/plugins/gh_v2.py | 69 --------------------- tests/core/usage_test.py | 1 - tests/main_test.py | 18 +----- tests/plugins/gh_test.py | 46 +++++++++++--- tests/plugins/gh_v2_test.py | 55 ---------------- tests/pre_commit_hook_test.py | 5 +- 9 files changed, 92 insertions(+), 162 deletions(-) delete mode 100644 detect_secrets/plugins/gh_v2.py delete mode 100644 tests/plugins/gh_v2_test.py diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index e5bfef46b..42926aab4 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -417,12 +417,6 @@ class PluginOptions: PluginDescriptor( classname='GHDetector', disable_flag_text='--no-gh-scan', - disable_help_text='Disable scanning for GH credentials', - is_default=False, - ), - PluginDescriptor( - classname='GHDetectorV2', - disable_flag_text='--no-gh-v2-scan', disable_help_text='Disable v2 scanner for GH credentials', is_default=True, ), diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py index ecc8c6dff..b88be0cba 100644 --- a/detect_secrets/plugins/common/initialize.py +++ b/detect_secrets/plugins/common/initialize.py @@ -9,7 +9,6 @@ from ..basic_auth import BasicAuthDetector # noqa: F401 from ..common.util import get_mapping_from_secret_type_to_class_name from ..gh import GHDetector # noqa: F401 -from ..gh_v2 import GHDetectorV2 # noqa: F401 from ..high_entropy_strings import Base64HighEntropyString # noqa: F401 from ..high_entropy_strings import HexHighEntropyString # noqa: F401 from ..keyword import KeywordDetector # noqa: F401 diff --git a/detect_secrets/plugins/gh.py b/detect_secrets/plugins/gh.py index 25766a1b2..76e242dd4 100644 --- a/detect_secrets/plugins/gh.py +++ b/detect_secrets/plugins/gh.py @@ -12,9 +12,58 @@ class GHDetector(RegexBasedDetector): secret_type = 'GitHub Credentials' + opt_github = r'(?:github|gh|ghe|git|)' + opt_space = r'(?: |)' + opt_quote = r'(?:"|\'|)' + opt_assignment = r'(?:=|:|:=|=>|)' + opt_dash_undrscr = r'(?:_|-|)' + opt_api = r'(?:api|)' + header_keyword = r'(?:token|bearer|Basic)' + key_or_pass = r'(?:key|pwd|password|pass|token|oauth)' + api_endpoint = r'(?:github.ibm.com|api.github.ibm.com)' + forty_hex = r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))' + b64_encoded_token = r'(?:(?<=\W)|(?<=^))([A-Za-z0-9+/]{55}=)(?:(?=\W)|(?=$))' + opt_username = r'(?:[a-zA-Z0-9-]+:|)' denylist = [ - # GitHub tokens (PAT & OAuth) are 40 hex characters - re.compile(r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))'), # 40 hex + re.compile( + r'{opt_quote}{opt_github}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}{key_or_pass}' + '{opt_quote}{opt_space}{opt_assignment}{opt_space}{opt_quote}{forty_hex}' + '{opt_quote}'.format( + opt_quote=opt_quote, + opt_github=opt_github, + opt_dash_undrscr=opt_dash_undrscr, + opt_api=opt_api, + key_or_pass=key_or_pass, + opt_space=opt_space, + opt_assignment=opt_assignment, + forty_hex=forty_hex, + ), flags=re.IGNORECASE, + ), + re.compile( + r'https://{opt_username}{forty_hex}@{api_endpoint}'.format( + forty_hex=forty_hex, + api_endpoint=api_endpoint, + opt_username=opt_username, + ), flags=re.IGNORECASE, + ), + re.compile( + r'{opt_quote}Authorization{opt_quote}{opt_space}:{opt_space}{opt_quote}' + '{header_keyword}{opt_space}{forty_hex}{opt_quote}'.format( + opt_quote=opt_quote, + opt_space=opt_space, + header_keyword=header_keyword, + forty_hex=forty_hex, + ), flags=re.IGNORECASE, + ), + re.compile( + r'{opt_quote}Authorization{opt_quote}{opt_space}:{opt_space}{opt_quote}' + 'Basic{opt_space}{b64_encoded_token}{opt_quote}'.format( + opt_quote=opt_quote, + opt_space=opt_space, + header_keyword=header_keyword, + b64_encoded_token=b64_encoded_token, + ), flags=re.IGNORECASE, + ), ] def verify(self, token, **kwargs): diff --git a/detect_secrets/plugins/gh_v2.py b/detect_secrets/plugins/gh_v2.py deleted file mode 100644 index 9abe42700..000000000 --- a/detect_secrets/plugins/gh_v2.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import absolute_import - -import re - -from .base import RegexBasedDetector -from detect_secrets.plugins.gh import GHDetector - - -class GHDetectorV2(RegexBasedDetector): - """ Tighter version of GHDetector. """ - - secret_type = 'GitHub Credentials' - - opt_github = r'(?:github|gh|ghe|git|)' - opt_space = r'(?: |)' - opt_quote = r'(?:"|\'|)' - opt_assignment = r'(?:=|:|:=|=>|)' - opt_dash_undrscr = r'(?:_|-|)' - opt_api = r'(?:api|)' - header_keyword = r'(?:token|bearer|Basic)' - key_or_pass = r'(?:key|pwd|password|pass|token|oauth)' - api_endpoint = r'(?:github.ibm.com|api.github.ibm.com)' - forty_hex = r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))' - b64_encoded_token = r'(?:(?<=\W)|(?<=^))([A-Za-z0-9+/]{55}=)(?:(?=\W)|(?=$))' - opt_username = r'(?:[a-zA-Z0-9-]+:|)' - denylist = [ - re.compile( - r'{opt_quote}{opt_github}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}{key_or_pass}' - '{opt_quote}{opt_space}{opt_assignment}{opt_space}{opt_quote}{forty_hex}' - '{opt_quote}'.format( - opt_quote=opt_quote, - opt_github=opt_github, - opt_dash_undrscr=opt_dash_undrscr, - opt_api=opt_api, - key_or_pass=key_or_pass, - opt_space=opt_space, - opt_assignment=opt_assignment, - forty_hex=forty_hex, - ), flags=re.IGNORECASE, - ), - re.compile( - r'https://{opt_username}{forty_hex}@{api_endpoint}'.format( - forty_hex=forty_hex, - api_endpoint=api_endpoint, - opt_username=opt_username, - ), flags=re.IGNORECASE, - ), - re.compile( - r'{opt_quote}Authorization{opt_quote}{opt_space}:{opt_space}{opt_quote}' - '{header_keyword}{opt_space}{forty_hex}{opt_quote}'.format( - opt_quote=opt_quote, - opt_space=opt_space, - header_keyword=header_keyword, - forty_hex=forty_hex, - ), flags=re.IGNORECASE, - ), - re.compile( - r'{opt_quote}Authorization{opt_quote}{opt_space}:{opt_space}{opt_quote}' - 'Basic{opt_space}{b64_encoded_token}{opt_quote}'.format( - opt_quote=opt_quote, - opt_space=opt_space, - header_keyword=header_keyword, - b64_encoded_token=b64_encoded_token, - ), flags=re.IGNORECASE, - ), - ] - - def verify(self, token, **kwargs): - return GHDetector().verify(token) diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py index 387ef8dff..82df84fbe 100644 --- a/tests/core/usage_test.py +++ b/tests/core/usage_test.py @@ -49,7 +49,6 @@ def test_consolidates_output_basic(self): 'StripeDetector': {}, 'ArtifactoryDetector': {}, 'GHDetector': {}, - 'GHDetectorV2': {}, 'SoftLayerDetector': {}, } >>>>>>> Define default plugin list diff --git a/tests/main_test.py b/tests/main_test.py index 3a5ddca67..cffbf9c41 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -194,7 +194,6 @@ def test_scan_string_basic( Base64HighEntropyString: {} BasicAuthDetector : False GHDetector : False - GHDetectorV2 : False HexHighEntropyString : {} KeywordDetector : False PrivateKeyDetector : False @@ -222,7 +221,7 @@ def test_scan_string_basic_default( AWSKeyDetector : False ArtifactoryDetector: False BasicAuthDetector : False - GHDetectorV2 : False + GHDetector : False PrivateKeyDetector : False SlackDetector : False SoftLayerDetector : False @@ -244,7 +243,6 @@ def test_scan_string_cli_overrides_stdin(self): Base64HighEntropyString: False (2.585) BasicAuthDetector : False GHDetector : False - GHDetectorV2 : False HexHighEntropyString : False (2.121) KeywordDetector : False PrivateKeyDetector : False @@ -381,9 +379,6 @@ def test_old_baseline_ignored_with_update_flag( { 'name': 'GHDetector', }, - { - 'name': 'GHDetectorV2', - }, { 'hex_limit': 3, 'name': 'HexHighEntropyString', @@ -427,9 +422,6 @@ def test_old_baseline_ignored_with_update_flag( { 'name': 'GHDetector', }, - { - 'name': 'GHDetectorV2', - }, { 'hex_limit': 3, 'name': 'HexHighEntropyString', @@ -530,9 +522,6 @@ def test_old_baseline_ignored_with_update_flag( { 'name': 'GHDetector', }, - { - 'name': 'GHDetectorV2', - }, { 'name': 'PrivateKeyDetector', }, @@ -575,9 +564,6 @@ def test_old_baseline_ignored_with_update_flag( { 'name': 'GHDetector', }, - { - 'name': 'GHDetectorV2', - }, { 'name': 'PrivateKeyDetector', }, @@ -718,7 +704,7 @@ def test_scan_with_default_plugin(self): 'name': 'BasicAuthDetector', }, { - 'name': 'GHDetectorV2', + 'name': 'GHDetector', }, { 'name': 'PrivateKeyDetector', diff --git a/tests/plugins/gh_test.py b/tests/plugins/gh_test.py index e439069a3..36aea5139 100644 --- a/tests/plugins/gh_test.py +++ b/tests/plugins/gh_test.py @@ -6,6 +6,7 @@ from detect_secrets.core.constants import VerifiedResult from detect_secrets.plugins.gh import GHDetector + GHE_TOKEN = 'abcdef0123456789abcdef0123456789abcdef01' GHE_TOKEN_BYTES = b'abcdef0123456789abcdef0123456789abcdef01' @@ -15,15 +16,42 @@ class TestGHDetector(object): @pytest.mark.parametrize( 'payload, should_flag', [ - ('2764d47e6bf540911b7da8fe55caa9451e783549', True), # not real key - ('key :53d49d5081266d939bac57a3d86c517ded974b19', True), # not real key - ('53d49dnotakeyata9bac57a3d86c517ded974b19', False), # has non-hex - ('a654fd9e3758a65235c765cf51e10df0c80b7a9', False), # only 39 - ('a654fd9e3758a65235c765cf51e10df0c80b7a923', False), # 41 - ('2764d47e6bf540911b7da8fe55caa9451e783549 ', True), # not real key - ('2764d47e6bf540911b7da8fe55caa9451e7835492 ', False), # not real key - ('2764d47e6bf540911b7da8fe55caa9451e783549_ ', False), # not real key - ('2764d47e6bf540911b7da8fe55caa9451e783549z ', False), # not real key + ('github-key 2764d47e6bf540911b7da8fe55caa9451e783549', True), + ('github_pwd :53d49d5081266d939bac57a3d86c517ded974b19', True), + ('gh-api-key=2764d47e6bf540911b7da8fe55caa9451e783549 ', True), + ('git-token => "abcdef0123456789abcdef0123456789abcdef01"', True), + ('"GHE_API_KEY": "abcdef0123456789abcdef0123456789abcdef01"', True), + ('GITHUB_API_TOKEN := "abcdef0123456789abcdef0123456789abcdef01"', True), + ('https://username:abcdef0123456789abcdef0123456789abcdef01@github.ibm.com', True,), + ( + 'https://username:abcdef0123456789abcdef0123456789abcdef01@' + 'api.github.ibm.com', True, + ), + ('Authorization: token abcdef0123456789abcdef0123456789abcdef01', True), + ( + 'Authorization: Basic ' + 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', True, + ), + ('password abcdef0123456789abcdef0123456789abcdef01', True,), + ('git+https://abcdef0123456789abcdef0123456789abcdef01@github.ibm.com', True,), + ('sonar.github.oauth=abcdef0123456789abcdef0123456789abcdef01', True,), + ( + 'https://x-oauth-basic:abcdef0123456789abcdef0123456789abcdef01' + '@github.ibm.com/org/repo.git', True, + ), + ('abcdef0123456789abcdef0123456789abcdef01', False), # no keyword prefix + ('gh-token=53d49dnotakeyata9bac57a3d86c517ded974b19', False), # has non-hex + ('GIT-KEY: a654fd9e3758a65235c765cf51e10df0c80b7a9', False), # only 39 + ('github_api_key: a654fd9e3758a65235c765cf51e10df0c80b7a923', False), # 41 + ('gh_key:=2764d47e6bf540911b7da8fe55caa9451e7835492 ', False), + ('github-api-token: 2764d47e6bf540911b7da8fe55caa9451e783549_ ', False), + ('git_key=2764d47e6bf540911b7da8fe55caa9451e783549z ', False), + ('https://:@github.ibm.com', False), + ( + 'Authorization: llama ' + 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', False, + ), + ('Authorization: token %s', False), ], ) def test_analyze_string(self, payload, should_flag): diff --git a/tests/plugins/gh_v2_test.py b/tests/plugins/gh_v2_test.py deleted file mode 100644 index 6b9c8fef4..000000000 --- a/tests/plugins/gh_v2_test.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import absolute_import - -import pytest - -from detect_secrets.plugins.gh_v2 import GHDetectorV2 - - -class TestGHDetectorV2(object): - - @pytest.mark.parametrize( - 'payload, should_flag', - [ - ('github-key 2764d47e6bf540911b7da8fe55caa9451e783549', True), - ('github_pwd :53d49d5081266d939bac57a3d86c517ded974b19', True), - ('gh-api-key=2764d47e6bf540911b7da8fe55caa9451e783549 ', True), - ('git-token => "abcdef0123456789abcdef0123456789abcdef01"', True), - ('"GHE_API_KEY": "abcdef0123456789abcdef0123456789abcdef01"', True), - ('GITHUB_API_TOKEN := "abcdef0123456789abcdef0123456789abcdef01"', True), - ('https://username:abcdef0123456789abcdef0123456789abcdef01@github.ibm.com', True,), - ( - 'https://username:abcdef0123456789abcdef0123456789abcdef01@' - 'api.github.ibm.com', True, - ), - ('Authorization: token abcdef0123456789abcdef0123456789abcdef01', True), - ( - 'Authorization: Basic ' - 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', True, - ), - ('password abcdef0123456789abcdef0123456789abcdef01', True,), - ('git+https://abcdef0123456789abcdef0123456789abcdef01@github.ibm.com', True,), - ('sonar.github.oauth=abcdef0123456789abcdef0123456789abcdef01', True,), - ( - 'https://x-oauth-basic:abcdef0123456789abcdef0123456789abcdef01' - '@github.ibm.com/org/repo.git', True, - ), - ('abcdef0123456789abcdef0123456789abcdef01', False), # no keyword prefix - ('gh-token=53d49dnotakeyata9bac57a3d86c517ded974b19', False), # has non-hex - ('GIT-KEY: a654fd9e3758a65235c765cf51e10df0c80b7a9', False), # only 39 - ('github_api_key: a654fd9e3758a65235c765cf51e10df0c80b7a923', False), # 41 - ('gh_key:=2764d47e6bf540911b7da8fe55caa9451e7835492 ', False), - ('github-api-token: 2764d47e6bf540911b7da8fe55caa9451e783549_ ', False), - ('git_key=2764d47e6bf540911b7da8fe55caa9451e783549z ', False), - ('https://:@github.ibm.com', False), - ( - 'Authorization: llama ' - 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', False, - ), - ('Authorization: token %s', False), - ], - ) - def test_analyze_string(self, payload, should_flag): - logic = GHDetectorV2() - - output = logic.analyze_string(payload, 1, 'mock_filename') - assert len(output) == int(should_flag) diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index 0aa9f8a6b..9f00404e5 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -200,12 +200,11 @@ def test_baseline_gets_updated( 'name': 'GHDetector', }, { -<<<<<<< HEAD -======= 'name': 'GHDetectorV2', }, { ->>>>>>> Use GHDetectorV2 (#181) +======= +>>>>>>> Delete GHDetector V1 (#186) 'hex_limit': 3, 'name': 'HexHighEntropyString', },