diff --git a/detect_secrets/filters/allowlist.py b/detect_secrets/filters/allowlist.py index 5cba1e3c6..db20c602c 100644 --- a/detect_secrets/filters/allowlist.py +++ b/detect_secrets/filters/allowlist.py @@ -58,18 +58,18 @@ def _get_allowlist_regexes_for_file(filename: str) -> Iterable[List[Pattern]]: comment_tuples = [comment_tuples[_get_file_to_index_dict()[ext[1:]]]] yield [ - _get_allowlist_regexes(comment_tuple=t, nextline=False) + get_allowlist_regexes(comment_tuple=t, nextline=False) for t in comment_tuples ] yield [ - _get_allowlist_regexes(comment_tuple=t, nextline=True) + get_allowlist_regexes(comment_tuple=t, nextline=True) for t in comment_tuples ] # Note: Cache size should be 2x the number of comment types @lru_cache(maxsize=12) -def _get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern: +def get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern: start = comment_tuple[0] end = comment_tuple[1] return re.compile( diff --git a/detect_secrets/transformers/config.py b/detect_secrets/transformers/config.py index 4a6f95ca1..9a5e3fcc2 100644 --- a/detect_secrets/transformers/config.py +++ b/detect_secrets/transformers/config.py @@ -12,6 +12,7 @@ from ..util.filetype import FileType from .base import BaseTransformer from .exceptions import ParsingError +from detect_secrets.filters.allowlist import get_allowlist_regexes class ConfigFileTransformer(BaseTransformer): @@ -49,6 +50,11 @@ def _parse_file(file: NamedIO, add_header: bool = False) -> List[str]: while len(lines) < line_number - 1: lines.append('') + # Always add 'pragma: allowlist nextline secret' comments + if _is_allowlist_nextline_secret_comment(value): + lines.append(value) + continue + # We artificially add quotes here because we know they are strings # (because it's a config file), HighEntropyString will benefit from this, # and all other plugins don't care. @@ -135,6 +141,16 @@ def _get_value_and_line_offset(self, key: str, values: str) -> List[Tuple[str, i output = [] for line_offset, line in enumerate(self.lines): + # Check 'pragma: allowlist nextline secret' comment on a single line + # The IniFileParser strips out comments however it is important to + # persist this speific comment type so filtering works properly. + if _is_allowlist_nextline_secret_comment(line): + output.append(( + line, + self.line_offset + line_offset + 1, + )) + continue + # Check ignored lines before checking values, because # you can write comments *after* the value. if not line or self._comment_regex.match(line): @@ -214,3 +230,14 @@ def _construct_values_list(values: str) -> List[str]: values_list = lines[:1] values_list.extend(filter(None, lines[1:])) return values_list + + +def _is_allowlist_nextline_secret_comment(line: str) -> bool: + # Valid tuples for config file comments (start_char, end_char) + comment_tuple = [('#', ''), (';', '')] + + for t in comment_tuple: + if get_allowlist_regexes(comment_tuple=t, nextline=True).search(line): + return True + + return False diff --git a/tests/transformers/config_transformer_test.py b/tests/transformers/config_transformer_test.py index 88b2349fe..311269901 100644 --- a/tests/transformers/config_transformer_test.py +++ b/tests/transformers/config_transformer_test.py @@ -42,6 +42,43 @@ def test_transformer(transformer): ] +@pytest.mark.parametrize( + 'transformer', + ( + ConfigFileTransformer, + EagerConfigFileTransformer, + ), +) +def test_transformer_persist_pragma_comments(transformer): + file = mock_file_object( + textwrap.dedent(""" + [section] + keyA = value + + # pragma: allowlist nextline secret + keyB = "double" + keyC = 'single' + + # pragma: allowlist nextline secret + keyD = o'brian + keyE = "chai" tea + """)[1:-1], + ) + + assert transformer().parse_file(file) == [ + '', + 'keyA = "value"', + '', + '# pragma: allowlist nextline secret', + 'keyB = "double"', + 'keyC = "single"', + '', + '# pragma: allowlist nextline secret', + 'keyD = "o\'brian"', + 'keyE = "\\\"chai\\\" tea"', + ] + + def test_basic(): file = mock_file_object( textwrap.dedent(""" @@ -66,6 +103,64 @@ def test_basic(): ] +def test_basic_persist_pragma_comments_pound(): + file = mock_file_object( + textwrap.dedent(""" + [section] + # pragma: allowlist nextline secret + key = value + # pragma: allowlist nextline secret + rice = fried + + # comment + tea = chai + + [other] + # pragma: allowlist nextline secret + water = unflavored + """)[1:-1], + ) + + assert list(IniFileParser(file)) == [ + ('key', '# pragma: allowlist nextline secret', 2), + ('key', 'value', 3), + ('key', '# pragma: allowlist nextline secret', 4), + ('rice', 'fried', 5), + ('tea', 'chai', 8), + ('water', '# pragma: allowlist nextline secret', 11), + ('water', 'unflavored', 12), + ] + + +def test_basic_persist_pragma_comments_semi_colon(): + file = mock_file_object( + textwrap.dedent(""" + [section] + ; pragma: allowlist nextline secret + key = value + ; pragma: allowlist nextline secret + rice = fried + + ; comment + tea = chai + + [other] + ; pragma: allowlist nextline secret + water = unflavored + """)[1:-1], + ) + + assert list(IniFileParser(file)) == [ + ('key', '; pragma: allowlist nextline secret', 2), + ('key', 'value', 3), + ('key', '; pragma: allowlist nextline secret', 4), + ('rice', 'fried', 5), + ('tea', 'chai', 8), + ('water', '; pragma: allowlist nextline secret', 11), + ('water', 'unflavored', 12), + ] + + @pytest.mark.parametrize( 'content', ( @@ -145,3 +240,26 @@ def test_not_first(): ('key', 'value1', 3), ('key', 'value2', 6), ] + + @staticmethod + def test_pragma_comments(): + file = mock_file_object( + textwrap.dedent(""" + [section] + # pragma: allowlist nextline secret + key = value0 + # pragma: allowlist nextline secret + value1 + + # comment + value2 + """)[1:-1], + ) + + assert list(IniFileParser(file)) == [ + ('key', '# pragma: allowlist nextline secret', 2), + ('key', 'value0', 3), + ('key', '# pragma: allowlist nextline secret', 4), + ('key', 'value1', 5), + ('key', 'value2', 8), + ]