Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Persist '# pragma: allowlist nextline secret' in IniFileParser For ConfigFileTransformers #575

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions detect_secrets/filters/allowlist.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,18 @@ def _get_allowlist_regexes_for_file(filename: str) -> Iterable[List[Pattern]]:
comment_tuples = [comment_tuples[_get_file_to_index_dict()[ext[1:]]]]

yield [
_get_allowlist_regexes(comment_tuple=t, nextline=False)
get_allowlist_regexes(comment_tuple=t, nextline=False)
for t in comment_tuples
]
yield [
_get_allowlist_regexes(comment_tuple=t, nextline=True)
get_allowlist_regexes(comment_tuple=t, nextline=True)
for t in comment_tuples
]


# Note: Cache size should be 2x the number of comment types
@lru_cache(maxsize=12)
def _get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern:
def get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern:
start = comment_tuple[0]
end = comment_tuple[1]
return re.compile(
Expand Down
27 changes: 27 additions & 0 deletions detect_secrets/transformers/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from ..util.filetype import FileType
from .base import BaseTransformer
from .exceptions import ParsingError
from detect_secrets.filters.allowlist import get_allowlist_regexes


class ConfigFileTransformer(BaseTransformer):
Expand Down Expand Up @@ -49,6 +50,11 @@ def _parse_file(file: NamedIO, add_header: bool = False) -> List[str]:
while len(lines) < line_number - 1:
lines.append('')

# Always add 'pragma: allowlist nextline secret' comments
if _is_allowlist_nextline_secret_comment(value):
lines.append(value)
continue

# We artificially add quotes here because we know they are strings
# (because it's a config file), HighEntropyString will benefit from this,
# and all other plugins don't care.
Expand Down Expand Up @@ -135,6 +141,16 @@ def _get_value_and_line_offset(self, key: str, values: str) -> List[Tuple[str, i
output = []

for line_offset, line in enumerate(self.lines):
# Check 'pragma: allowlist nextline secret' comment on a single line
# The IniFileParser strips out comments however it is important to
# persist this speific comment type so filtering works properly.
if _is_allowlist_nextline_secret_comment(line):
output.append((
line,
self.line_offset + line_offset + 1,
))
continue

# Check ignored lines before checking values, because
# you can write comments *after* the value.
if not line or self._comment_regex.match(line):
Expand Down Expand Up @@ -214,3 +230,14 @@ def _construct_values_list(values: str) -> List[str]:
values_list = lines[:1]
values_list.extend(filter(None, lines[1:]))
return values_list


def _is_allowlist_nextline_secret_comment(line: str) -> bool:
# Valid tuples for config file comments (start_char, end_char)
comment_tuple = [('#', ''), (';', '')]

for t in comment_tuple:
if get_allowlist_regexes(comment_tuple=t, nextline=True).search(line):
return True

return False
118 changes: 118 additions & 0 deletions tests/transformers/config_transformer_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,43 @@ def test_transformer(transformer):
]


@pytest.mark.parametrize(
'transformer',
(
ConfigFileTransformer,
EagerConfigFileTransformer,
),
)
def test_transformer_persist_pragma_comments(transformer):
file = mock_file_object(
textwrap.dedent("""
[section]
keyA = value

# pragma: allowlist nextline secret
keyB = "double"
keyC = 'single'

# pragma: allowlist nextline secret
keyD = o'brian
keyE = "chai" tea
""")[1:-1],
)

assert transformer().parse_file(file) == [
'',
'keyA = "value"',
'',
'# pragma: allowlist nextline secret',
'keyB = "double"',
'keyC = "single"',
'',
'# pragma: allowlist nextline secret',
'keyD = "o\'brian"',
'keyE = "\\\"chai\\\" tea"',
]


def test_basic():
file = mock_file_object(
textwrap.dedent("""
Expand All @@ -66,6 +103,64 @@ def test_basic():
]


def test_basic_persist_pragma_comments_pound():
file = mock_file_object(
textwrap.dedent("""
[section]
# pragma: allowlist nextline secret
key = value
# pragma: allowlist nextline secret
rice = fried

# comment
tea = chai

[other]
# pragma: allowlist nextline secret
water = unflavored
""")[1:-1],
)

assert list(IniFileParser(file)) == [
('key', '# pragma: allowlist nextline secret', 2),
('key', 'value', 3),
('key', '# pragma: allowlist nextline secret', 4),
('rice', 'fried', 5),
('tea', 'chai', 8),
('water', '# pragma: allowlist nextline secret', 11),
('water', 'unflavored', 12),
]


def test_basic_persist_pragma_comments_semi_colon():
file = mock_file_object(
textwrap.dedent("""
[section]
; pragma: allowlist nextline secret
key = value
; pragma: allowlist nextline secret
rice = fried

; comment
tea = chai

[other]
; pragma: allowlist nextline secret
water = unflavored
""")[1:-1],
)

assert list(IniFileParser(file)) == [
('key', '; pragma: allowlist nextline secret', 2),
('key', 'value', 3),
('key', '; pragma: allowlist nextline secret', 4),
('rice', 'fried', 5),
('tea', 'chai', 8),
('water', '; pragma: allowlist nextline secret', 11),
('water', 'unflavored', 12),
]


@pytest.mark.parametrize(
'content',
(
Expand Down Expand Up @@ -145,3 +240,26 @@ def test_not_first():
('key', 'value1', 3),
('key', 'value2', 6),
]

@staticmethod
def test_pragma_comments():
file = mock_file_object(
textwrap.dedent("""
[section]
# pragma: allowlist nextline secret
key = value0
# pragma: allowlist nextline secret
value1

# comment
value2
""")[1:-1],
)

assert list(IniFileParser(file)) == [
('key', '# pragma: allowlist nextline secret', 2),
('key', 'value0', 3),
('key', '# pragma: allowlist nextline secret', 4),
('key', 'value1', 5),
('key', 'value2', 8),
]