Persist '# pragma: allowlist nextline secret' in IniFileParser For Co…

…nfigFileTransformers (#575)

* Persist pragma allowlist nextline comments in IniFileParser so filters work properly

* Add missing type annotations

detect_secrets/filters/allowlist.py

@@ -58,18 +58,18 @@ def _get_allowlist_regexes_for_file(filename: str) -> Iterable[List[Pattern]]:
         comment_tuples = [comment_tuples[_get_file_to_index_dict()[ext[1:]]]]
 
     yield [
-        _get_allowlist_regexes(comment_tuple=t, nextline=False)
+        get_allowlist_regexes(comment_tuple=t, nextline=False)
         for t in comment_tuples
     ]
     yield [
-        _get_allowlist_regexes(comment_tuple=t, nextline=True)
+        get_allowlist_regexes(comment_tuple=t, nextline=True)
         for t in comment_tuples
     ]
 
 
 # Note: Cache size should be 2x the number of comment types
 @lru_cache(maxsize=12)
-def _get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern:
+def get_allowlist_regexes(comment_tuple: Tuple[str, str], nextline: bool) -> Pattern:
     start = comment_tuple[0]
     end = comment_tuple[1]
     return re.compile(

detect_secrets/transformers/config.py

@@ -12,6 +12,7 @@
 from ..util.filetype import FileType
 from .base import BaseTransformer
 from .exceptions import ParsingError
+from detect_secrets.filters.allowlist import get_allowlist_regexes
 
 
 class ConfigFileTransformer(BaseTransformer):
@@ -49,6 +50,11 @@ def _parse_file(file: NamedIO, add_header: bool = False) -> List[str]:
         while len(lines) < line_number - 1:
             lines.append('')
 
+        # Always add 'pragma: allowlist nextline secret' comments
+        if _is_allowlist_nextline_secret_comment(value):
+            lines.append(value)
+            continue
+
         # We artificially add quotes here because we know they are strings
         # (because it's a config file), HighEntropyString will benefit from this,
         # and all other plugins don't care.
@@ -135,6 +141,16 @@ def _get_value_and_line_offset(self, key: str, values: str) -> List[Tuple[str, i
         output = []
 
         for line_offset, line in enumerate(self.lines):
+            # Check 'pragma: allowlist nextline secret' comment on a single line
+            # The IniFileParser strips out comments however it is important to
+            # persist this speific comment type so filtering works properly.
+            if _is_allowlist_nextline_secret_comment(line):
+                output.append((
+                    line,
+                    self.line_offset + line_offset + 1,
+                ))
+                continue
+
             # Check ignored lines before checking values, because
             # you can write comments *after* the value.
             if not line or self._comment_regex.match(line):
@@ -214,3 +230,14 @@ def _construct_values_list(values: str) -> List[str]:
     values_list = lines[:1]
     values_list.extend(filter(None, lines[1:]))
     return values_list
+
+
+def _is_allowlist_nextline_secret_comment(line: str) -> bool:
+    # Valid tuples for config file comments (start_char, end_char)
+    comment_tuple = [('#', ''), (';', '')]
+
+    for t in comment_tuple:
+        if get_allowlist_regexes(comment_tuple=t, nextline=True).search(line):
+            return True
+
+    return False

tests/transformers/config_transformer_test.py

@@ -42,6 +42,43 @@ def test_transformer(transformer):
     ]
 
 
+@pytest.mark.parametrize(
+    'transformer',
+    (
+        ConfigFileTransformer,
+        EagerConfigFileTransformer,
+    ),
+)
+def test_transformer_persist_pragma_comments(transformer):
+    file = mock_file_object(
+        textwrap.dedent("""
+            [section]
+            keyA = value
+
+            # pragma: allowlist nextline secret
+            keyB = "double"
+            keyC = 'single'
+
+            # pragma: allowlist nextline secret
+            keyD = o'brian
+            keyE = "chai" tea
+        """)[1:-1],
+    )
+
+    assert transformer().parse_file(file) == [
+        '',
+        'keyA = "value"',
+        '',
+        '# pragma: allowlist nextline secret',
+        'keyB = "double"',
+        'keyC = "single"',
+        '',
+        '# pragma: allowlist nextline secret',
+        'keyD = "o\'brian"',
+        'keyE = "\\\"chai\\\" tea"',
+    ]
+
+
 def test_basic():
     file = mock_file_object(
         textwrap.dedent("""
@@ -66,6 +103,64 @@ def test_basic():
     ]
 
 
+def test_basic_persist_pragma_comments_pound():
+    file = mock_file_object(
+        textwrap.dedent("""
+            [section]
+            # pragma: allowlist nextline secret
+            key = value
+            # pragma: allowlist nextline secret
+            rice = fried
+
+            # comment
+            tea = chai
+
+            [other]
+            # pragma: allowlist nextline secret
+            water = unflavored
+        """)[1:-1],
+    )
+
+    assert list(IniFileParser(file)) == [
+        ('key', '# pragma: allowlist nextline secret', 2),
+        ('key', 'value', 3),
+        ('key', '# pragma: allowlist nextline secret', 4),
+        ('rice', 'fried', 5),
+        ('tea', 'chai', 8),
+        ('water', '# pragma: allowlist nextline secret', 11),
+        ('water', 'unflavored', 12),
+    ]
+
+
+def test_basic_persist_pragma_comments_semi_colon():
+    file = mock_file_object(
+        textwrap.dedent("""
+            [section]
+            ; pragma: allowlist nextline secret
+            key = value
+            ; pragma: allowlist nextline secret
+            rice = fried
+
+            ; comment
+            tea = chai
+
+            [other]
+            ; pragma: allowlist nextline secret
+            water = unflavored
+        """)[1:-1],
+    )
+
+    assert list(IniFileParser(file)) == [
+        ('key', '; pragma: allowlist nextline secret', 2),
+        ('key', 'value', 3),
+        ('key', '; pragma: allowlist nextline secret', 4),
+        ('rice', 'fried', 5),
+        ('tea', 'chai', 8),
+        ('water', '; pragma: allowlist nextline secret', 11),
+        ('water', 'unflavored', 12),
+    ]
+
+
 @pytest.mark.parametrize(
     'content',
     (
@@ -145,3 +240,26 @@ def test_not_first():
             ('key', 'value1', 3),
             ('key', 'value2', 6),
         ]
+
+    @staticmethod
+    def test_pragma_comments():
+        file = mock_file_object(
+            textwrap.dedent("""
+                [section]
+                # pragma: allowlist nextline secret
+                key = value0
+                    # pragma: allowlist nextline secret
+                    value1
+
+                    # comment
+                    value2
+            """)[1:-1],
+        )
+
+        assert list(IniFileParser(file)) == [
+            ('key', '# pragma: allowlist nextline secret', 2),
+            ('key', 'value0', 3),
+            ('key', '# pragma: allowlist nextline secret', 4),
+            ('key', 'value1', 5),
+            ('key', 'value2', 8),
+        ]