diff --git a/detect_secrets/plugins/common/filetype.py b/detect_secrets/plugins/common/filetype.py index 883ab52d1..687805b56 100644 --- a/detect_secrets/plugins/common/filetype.py +++ b/detect_secrets/plugins/common/filetype.py @@ -1,15 +1,38 @@ +import os from enum import Enum class FileType(Enum): CLS = 0 - GO = 1 - JAVA = 2 - JAVASCRIPT = 3 - PHP = 4 - PYTHON = 5 - YAML = 6 - OTHER = 7 + EXAMPLE = 1 + GO = 2 + JAVA = 3 + JAVASCRIPT = 4 + PHP = 5 + OBJECTIVE_C = 6 + PYTHON = 7 + SWIFT = 8 + TERRAFORM = 9 + YAML = 10 + OTHER = 11 + + +EXTENSION_TO_FILETYPE = { + '.cls': FileType.CLS, + '.example': FileType.EXAMPLE, + '.eyaml': FileType.YAML, + '.go': FileType.GO, + '.java': FileType.JAVA, + '.js': FileType.JAVASCRIPT, + '.m': FileType.OBJECTIVE_C, + '.php': FileType.PHP, + '.py': FileType.PYTHON, + '.pyi': FileType.PYTHON, + '.swift': FileType.SWIFT, + '.tf': FileType.TERRAFORM, + '.yaml': FileType.YAML, + '.yml': FileType.YAML, +} def determine_file_type(filename): @@ -18,22 +41,8 @@ def determine_file_type(filename): :rtype: FileType """ - if filename.endswith('.cls'): - return FileType.CLS - elif filename.endswith('.go'): - return FileType.GO - elif filename.endswith('.java'): - return FileType.JAVA - elif filename.endswith('.js'): - return FileType.JAVASCRIPT - elif filename.endswith('.php'): - return FileType.PHP - elif filename.endswith('.py'): - return FileType.PYTHON - elif ( - filename.endswith( - ('.eyaml', '.yaml', '.yml'), - ) - ): - return FileType.YAML - return FileType.OTHER + _, file_extension = os.path.splitext(filename) + return EXTENSION_TO_FILETYPE.get( + file_extension, + FileType.OTHER, + ) diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index 5716c3381..e9cca6bc7 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -66,8 +66,9 @@ "'this", '(nsstring', '-default}', - '/etc/passwd:ro', '::', + '<%=', + '', '= 3 + # For e.g. "secret": "{secret}" + or ( + lowered_secret[0] == '{' + and lowered_secret[-1] == '}' ) or ( - filetype == FileType.PHP + filetype not in QUOTES_REQUIRED_FILETYPES and lowered_secret[0] == '$' ) or ( - filetype == FileType.YAML - and lowered_secret.startswith('{{') - and lowered_secret.endswith('}}') + filetype == FileType.EXAMPLE + and lowered_secret[0] == '<' + and lowered_secret[-1] == '>' ) ): return True diff --git a/tests/plugins/keyword_test.py b/tests/plugins/keyword_test.py index a7f2759cf..3b7eb4bf0 100644 --- a/tests/plugins/keyword_test.py +++ b/tests/plugins/keyword_test.py @@ -8,11 +8,37 @@ from testing.mocks import mock_file_object -QUOTES_REQUIRED_FILE_EXTENSIONS = ( - '.cls', - '.java', - '.py', -) +FOLLOWED_BY_COLON_EQUAL_SIGNS_RE = { + 'negatives': { + 'quotes_required': [ + 'theapikey := ""', # Nothing in the quotes + 'theapikey := "somefakekey"', # 'fake' in the secret + ], + 'quotes_not_required': [ + 'theapikeyforfoo := hopenobodyfindsthisone', # Characters between apikey and := + ], + }, + 'positives': { + 'quotes_required': [ + 'apikey := "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey :="m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey := "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + "apikey := 'm{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + "apikey :='m{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + 'apikey:= "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey:="m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + "apikey:= 'm{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + "apikey:='m{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + "apikey:= 'm{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + ], + 'quotes_not_required': [ + 'apikey := m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey :=m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey:= m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey:=m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + ], + }, +} FOLLOWED_BY_COLON_RE = { 'negatives': { 'quotes_required': [ @@ -26,15 +52,35 @@ }, 'positives': { 'quotes_required': [ - "'theapikey': '{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - '"theapikey": "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - 'apikey: "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - "apikey: '{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + "'theapikey': 'm{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", + '"theapikey": "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey: "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + "apikey: 'm{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", ], 'quotes_not_required': [ - 'apikey: {{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'apikey:{{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'theapikey:{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey: m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey:m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'theapikey:m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + ], + }, +} +FOLLOWED_BY_EQUAL_SIGNS_OPTIONAL_BRACKETS_OPTIONAL_AT_SIGN_QUOTES_REQUIRED_REGEX = { + 'negatives': { + 'quotes_required': [ + 'theapikey[] = ""', # Nothing in the quotes + 'theapikey = @"somefakekey"', # 'fake' in the secret + ], + }, + 'positives': { + 'quotes_required': [ + 'apikey = "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey ="m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey = "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey = @"m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey =@"m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey = @"m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey[]= "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'apikey[]="m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', ], }, } @@ -57,18 +103,18 @@ }, 'positives': { 'quotes_required': [ - 'some_dict["secret"] = "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - 'the_password= "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"\n', - 'the_password=\'{{h}o)p${e]nob(ody[finds>-_$#thisone}}\'\n', + 'some_dict["secret"] = "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', + 'the_password= "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}"\n', + 'the_password=\'m{{h}o)p${e]nob(ody[finds>-_$#thisone}}\'\n', ], 'quotes_not_required': [ - "some_dict['secret'] = {{h}o)p${e]nob(ody[finds>-_$#thisone}}", - 'my_password={{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'my_password= {{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'my_password ={{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'my_password = {{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'my_password ={{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'the_password={{h}o)p${e]nob(ody[finds>-_$#thisone}}\n', + "some_dict['secret'] = m{{h}o)p${e]nob(ody[finds>-_$#thisone}}", + 'my_password=m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'my_password= m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'my_password =m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'my_password = m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'my_password =m{{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'the_password=m{{h}o)p${e]nob(ody[finds>-_$#thisone}}\n', ], }, } @@ -78,64 +124,46 @@ 'private_key "";', # Nothing in the quotes 'private_key \'"no spaces\';', # Has whitespace in the secret 'private_key "fake";', # 'fake' in the secret + 'private_key "some/dir/aint/a/secret";', # 3 or more / + 'private_key "${FOO}";', # Starts with ${ and ends with } 'private_key "hopenobodyfindsthisone\';', # Double-quote does not match single-quote 'private_key \'hopenobodyfindsthisone";', # Single-quote does not match double-quote ], }, 'positives': { 'quotes_required': [ - 'apikey "{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes - 'fooapikeyfoo "{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes - 'fooapikeyfoo"{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes - 'private_key \'{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes - 'fooprivate_keyfoo\'{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes - 'fooprivate_key\'{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes - ], - }, -} -FOLLOWED_BY_COLON_EQUAL_SIGNS_RE = { - 'negatives': { - 'quotes_required': [ - 'theapikey := ""', # Nothing in the quotes - 'theapikey := "somefakekey"', # 'fake' in the secret - ], - 'quotes_not_required': [ - 'theapikeyforfoo := hopenobodyfindsthisone', # Characters between apikey and := - ], - }, - 'positives': { - 'quotes_required': [ - 'apikey := "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - 'apikey :="{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - 'apikey := "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - "apikey := '{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - "apikey :='{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - 'apikey:= "{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - 'apikey:="{{h}o)p${e]nob(ody[finds>-_$#thisone}}"', - "apikey:= '{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - "apikey:='{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - "apikey:= '{{h}o)p${e]nob(ody[finds>-_$#thisone}}'", - ], - 'quotes_not_required': [ - 'apikey := {{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'apikey :={{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'apikey:= {{h}o)p${e]nob(ody[finds>-_$#thisone}}', - 'apikey:={{h}o)p${e]nob(ody[finds>-_$#thisone}}', + 'apikey "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes + 'fooapikeyfoo "m{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes + 'fooapikeyfoo"m{{h}o)p${e]nob(ody[finds>-_$#thisone}}";', # Double-quotes + 'private_key \'m{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes + 'fooprivate_keyfoo\'m{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes + 'fooprivate_key\'m{{h}o)p${e]nob(ody[finds>-_$#thisone}}\';', # Single-quotes ], }, } +QUOTES_REQUIRED_FILE_EXTENSIONS = ( + '.cls', + '.java', + '.js', + '.py', + '.swift', +) + STANDARD_NEGATIVES = [] STANDARD_POSITIVES = [] STANDARD_NEGATIVES.extend( - FOLLOWED_BY_COLON_RE.get('negatives').get('quotes_required') + FOLLOWED_BY_COLON_EQUAL_SIGNS_RE.get('negatives').get('quotes_required') + + FOLLOWED_BY_COLON_EQUAL_SIGNS_RE.get('negatives').get('quotes_not_required') + + FOLLOWED_BY_COLON_RE.get('negatives').get('quotes_required') + FOLLOWED_BY_COLON_RE.get('negatives').get('quotes_not_required') + FOLLOWED_BY_EQUAL_SIGNS_RE.get('negatives').get('quotes_required') + FOLLOWED_BY_EQUAL_SIGNS_RE.get('negatives').get('quotes_not_required') + FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE.get('negatives').get('quotes_required') - + FOLLOWED_BY_COLON_EQUAL_SIGNS_RE.get('negatives').get('quotes_required') - + FOLLOWED_BY_COLON_EQUAL_SIGNS_RE.get('negatives').get('quotes_not_required'), + + FOLLOWED_BY_EQUAL_SIGNS_OPTIONAL_BRACKETS_OPTIONAL_AT_SIGN_QUOTES_REQUIRED_REGEX.get( + 'negatives', + ).get('quotes_required'), ) STANDARD_POSITIVES.extend( FOLLOWED_BY_COLON_RE.get('positives').get('quotes_required') @@ -162,7 +190,7 @@ def test_analyze_standard_positives(self, file_content): assert 'mock_filename' == potential_secret.filename assert ( potential_secret.secret_hash - == PotentialSecret.hash_secret('{{h}o)p${e]nob(ody[finds>-_$#thisone}}') + == PotentialSecret.hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}') ) @pytest.mark.parametrize( @@ -198,7 +226,7 @@ def test_analyze_quotes_required_positives(self, file_content, file_extension): assert mock_filename == potential_secret.filename assert ( potential_secret.secret_hash - == PotentialSecret.hash_secret('{{h}o)p${e]nob(ody[finds>-_$#thisone}}') + == PotentialSecret.hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}') ) @pytest.mark.parametrize( @@ -219,7 +247,26 @@ def test_analyze_go_positives(self, file_content): assert 'mock_filename.go' == potential_secret.filename assert ( potential_secret.secret_hash == - PotentialSecret.hash_secret('{{h}o)p${e]nob(ody[finds>-_$#thisone}}') + PotentialSecret.hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}') + ) + + @pytest.mark.parametrize( + 'file_content', + FOLLOWED_BY_EQUAL_SIGNS_OPTIONAL_BRACKETS_OPTIONAL_AT_SIGN_QUOTES_REQUIRED_REGEX.get( + 'positives', + ).get('quotes_required'), + ) + def test_analyze_objective_c_positives(self, file_content): + logic = KeywordDetector() + + f = mock_file_object(file_content) + output = logic.analyze(f, 'mock_filename.m') + assert len(output) == 1 + for potential_secret in output: + assert 'mock_filename.m' == potential_secret.filename + assert ( + potential_secret.secret_hash == + PotentialSecret.hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}') ) @pytest.mark.parametrize( @@ -297,8 +344,8 @@ def test_analyze_quotes_required_negatives(self, file_content, file_extension): @pytest.mark.parametrize( 'file_content, file_extension', ( - (yaml_negative, file_extension) - for yaml_negative in STANDARD_POSITIVES + (standard_positive, file_extension) + for standard_positive in STANDARD_POSITIVES for file_extension in ( '.yaml', '.yml', @@ -308,9 +355,27 @@ def test_analyze_quotes_required_negatives(self, file_content, file_extension): def test_analyze_yaml_negatives(self, file_content, file_extension): logic = KeywordDetector() - f = mock_file_object(file_content) + # Make it start with `{{`, (and end with `}}`) so it hits our false-positive check + f = mock_file_object(file_content.replace('m{', '{')) output = logic.analyze( f, 'mock_filename{}'.format(file_extension), ) assert len(output) == 0 + + @pytest.mark.parametrize( + 'file_content', + STANDARD_POSITIVES, + ) + def test_analyze_example_negatives(self, file_content): + logic = KeywordDetector() + + # Make it start with `<`, (and end with `>`) so it hits our false-positive check + f = mock_file_object( + file_content.replace('m{', '<').replace('}', '>'), + ) + output = logic.analyze( + f, + 'mock_filename.example', + ) + assert len(output) == 0