Merge pull request #586 from eddiez9/feature/support-multi-os-path

Feature/support multi os baseline files

detect_secrets/core/potential_secret.py

@@ -6,6 +6,7 @@
 
 from ..util.color import AnsiColor
 from ..util.color import colorize
+from ..util.path import convert_local_os_path
 
 
 class PotentialSecret:
@@ -75,7 +76,7 @@ def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'Poten
         """Custom JSON decoder"""
         kwargs: Dict[str, Any] = {
             'type': str(data['type']),
-            'filename': str(data['filename']),
+            'filename': convert_local_os_path(str(data['filename'])),
             'secret': 'will be replaced',
         }
 

detect_secrets/core/secrets_collection.py

@@ -10,6 +10,7 @@
 from typing import Tuple
 
 from . import scan
+from ..util.path import convert_local_os_path
 from .potential_secret import PotentialSecret
 from detect_secrets.settings import configure_settings_from_baseline
 from detect_secrets.settings import get_settings
@@ -40,7 +41,7 @@ def load_from_baseline(cls, baseline: Dict[str, Any]) -> 'SecretsCollection':
         for filename in baseline['results']:
             for item in baseline['results'][filename]:
                 secret = PotentialSecret.load_secret_from_dict({'filename': filename, **item})
-                output[filename].add(secret)
+                output[convert_local_os_path(filename)].add(secret)
 
         return output
 
@@ -72,8 +73,8 @@ def scan_files(self, *filenames: str, num_processors: Optional[int] = None) -> N
                     self[os.path.relpath(secret.filename, self.root)].add(secret)
 
     def scan_file(self, filename: str) -> None:
-        for secret in scan.scan_file(os.path.join(self.root, filename)):
-            self[filename].add(secret)
+        for secret in scan.scan_file(os.path.join(self.root, convert_local_os_path(filename))):
+            self[convert_local_os_path(filename)].add(secret)
 
     def scan_diff(self, diff: str) -> None:
         """

detect_secrets/util/path.py

@@ -16,3 +16,13 @@ def get_relative_path_if_in_cwd(path: str) -> Optional[str]:
         return filepath
 
     return None
+
+
+def convert_local_os_path(path: str) -> str:
+    # Linux filesystem, replace \\ with /
+    if os.sep == '/':
+        path = path.replace('\\', '/')
+        return path
+    else:
+        path = path.replace('/', '\\')
+        return path

tests/audit/audit_test.py

@@ -1,5 +1,6 @@
 import json
 import random
+from pathlib import Path
 from typing import List
 from typing import Optional
 from unittest import mock
@@ -135,7 +136,7 @@ def test_ensure_file_transformers_are_used(printer):
         run_logic(secrets, 'y')
         assert not m.called
 
-    line_number = list(secrets['test_data/config.env'])[0].line_number
+    line_number = list(secrets[str(Path('test_data/config.env'))])[0].line_number
     assert lines[line_number - 1] in printer.message
 
 

tests/core/baseline_test.py

@@ -1,4 +1,5 @@
 import json
+import os
 import subprocess
 import tempfile
 from pathlib import Path
@@ -84,16 +85,21 @@ def test_scan_all_files():
 
 def test_load_and_output():
     with open('.secrets.baseline') as f:
-        data = json.loads(f.read())
+        filedata = f.read()
 
-    secrets = baseline.load(data, filename='.secrets.baseline')
+    if os.sep == '\\':
+        # Replace Linux path seperators for Windows ones
+        filedata = filedata.replace('/', '\\\\')
+
+    filedata_json = json.loads(filedata)
+    secrets = baseline.load(filedata_json, filename='.secrets.baseline')
     output = baseline.format_for_output(secrets)
 
-    for item in [data, output]:
+    for item in [filedata_json, output]:
         item.pop('generated_at')
 
     # We perform string matching because we want to ensure stable sorts.
-    assert json.dumps(output) == json.dumps(data)
+    assert json.dumps(output) == json.dumps(filedata_json)
 
     # We need to make sure that default values carry through, for future backwards compatibility.
     for plugin in output['plugins_used']:

tests/core/secrets_collection_test.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from unittest import mock
 
 import pytest
@@ -43,7 +44,8 @@ def test_error_reading_file(mock_log_warning):
         ):
             SecretsCollection().scan_file('test_data/config.env')
 
-        assert 'Unable to open file: test_data/config.env' in mock_log_warning.warning_messages
+        file_warning = str(Path('test_data/config.env'))
+        assert 'Unable to open file: %s' % file_warning in mock_log_warning.warning_messages
 
     @staticmethod
     def test_line_based_success():
@@ -64,9 +66,9 @@ def test_line_based_success():
         secrets = SecretsCollection()
         secrets.scan_file('test_data/each_secret.py')
 
-        secret = next(iter(secrets['test_data/each_secret.py']))
+        secret = next(iter(secrets[str(Path('test_data/each_secret.py'))]))
         assert secret.secret_value.startswith('c2VjcmV0IG1lc')
-        assert len(secrets['test_data/each_secret.py']) == 1
+        assert len(secrets[str(Path('test_data/each_secret.py'))]) == 1
 
     @staticmethod
     def test_file_based_success_config():
@@ -80,11 +82,11 @@ def test_file_based_success_config():
         secrets.scan_file('test_data/config.ini')
 
         assert [str(secret).splitlines()[1] for _, secret in secrets] == [
-            'Location:    test_data/config.ini:2',
-            'Location:    test_data/config.ini:10',
-            'Location:    test_data/config.ini:21',
-            'Location:    test_data/config.ini:22',
-            'Location:    test_data/config.ini:32',
+            'Location:    %s:2' % str(Path('test_data/config.ini')),
+            'Location:    %s:10' % str(Path('test_data/config.ini')),
+            'Location:    %s:21' % str(Path('test_data/config.ini')),
+            'Location:    %s:22' % str(Path('test_data/config.ini')),
+            'Location:    %s:32' % str(Path('test_data/config.ini')),
         ]
 
     @staticmethod
@@ -99,9 +101,9 @@ def test_file_based_success_yaml():
         secrets.scan_file('test_data/config.yaml')
 
         assert [str(secret).splitlines()[1] for _, secret in secrets] == [
-            'Location:    test_data/config.yaml:3',
-            'Location:    test_data/config.yaml:5',
-            'Location:    test_data/config.yaml:13',
+            'Location:    %s:3' % str(Path('test_data/config.yaml')),
+            'Location:    %s:5' % str(Path('test_data/config.yaml')),
+            'Location:    %s:13' % str(Path('test_data/config.yaml')),
         ]
 
     @staticmethod
@@ -217,12 +219,12 @@ def test_deleted_secret():
         secrets.scan_file('test_data/each_secret.py')
 
         results = SecretsCollection.load_from_baseline({'results': secrets.json()})
-        results.data['test_data/each_secret.py'].pop()
+        results.data[str(Path('test_data/each_secret.py'))].pop()
 
-        original_size = len(secrets['test_data/each_secret.py'])
+        original_size = len(secrets[str(Path('test_data/each_secret.py'))])
         secrets.trim(results)
 
-        assert len(secrets['test_data/each_secret.py']) < original_size
+        assert len(secrets[str(Path('test_data/each_secret.py'))]) < original_size
 
     @staticmethod
     def test_deleted_secret_file():
@@ -232,7 +234,7 @@ def test_deleted_secret_file():
         secrets.trim(SecretsCollection())
         assert secrets
 
-        secrets.trim(SecretsCollection(), filelist=['test_data/each_secret.py'])
+        secrets.trim(SecretsCollection(), filelist=[str(Path('test_data/each_secret.py'))])
         assert not secrets
 
     @staticmethod
@@ -288,7 +290,7 @@ def test_remove_non_existent_files():
         secrets.scan_file('test_data/each_secret.py')
         assert bool(secrets)
 
-        secrets.data['does-not-exist'] = secrets.data.pop('test_data/each_secret.py')
+        secrets.data['does-not-exist'] = secrets.data.pop(str(Path('test_data/each_secret.py')))
         secrets.trim()
 
         assert not bool(secrets)
@@ -316,7 +318,7 @@ def test_bool():
     secrets.scan_file('test_data/each_secret.py')
     assert secrets
 
-    secrets['test_data/each_secret.py'].clear()
+    secrets[str(Path('test_data/each_secret.py'))].clear()
     assert not secrets
 
 
@@ -373,8 +375,8 @@ def test_basic(configure_plugins):
         assert secrets != baseline
 
         result = secrets - baseline
-        assert len(result['test_data/each_secret.py']) == 3
-        assert len(secrets['test_data/each_secret.py']) == 5
+        assert len(result[str(Path('test_data/each_secret.py'))]) == 3
+        assert len(secrets[str(Path('test_data/each_secret.py'))]) == 5
 
     @staticmethod
     def test_no_overlapping_files(configure_plugins):
@@ -384,5 +386,5 @@ def test_no_overlapping_files(configure_plugins):
             secrets_a.scan_file('test_data/each_secret.py')
             secrets_b.scan_file('test_data/config.env')
 
-        assert (secrets_a - secrets_b).files == {'test_data/each_secret.py'}
-        assert (secrets_b - secrets_a).files == {'test_data/config.env'}
+        assert (secrets_a - secrets_b).files == {str(Path('test_data/each_secret.py'))}
+        assert (secrets_b - secrets_a).files == {str(Path('test_data/config.env'))}

tests/pre_commit_hook_test.py

@@ -3,6 +3,7 @@
 import sys
 from contextlib import contextmanager
 from functools import partial
+from pathlib import Path
 from typing import List
 from unittest import mock
 
@@ -68,7 +69,7 @@ def test_baseline_filters_out_known_secrets():
             ])
 
         # Remove one arbitrary secret, so that it won't be the full set.
-        secrets.data['test_data/each_secret.py'].pop()
+        secrets.data[str(Path('test_data/each_secret.py'))].pop()
 
         with mock_named_temporary_file() as f:
             baseline.save_to_file(secrets, f.name)
@@ -135,7 +136,7 @@ def test_success(self):
 
     def test_maintains_labelled_data(self):
         def label_secret(secrets):
-            list(secrets[self.FILENAME])[0].is_secret = True
+            list(secrets[str(Path(self.FILENAME))])[0].is_secret = True
             return baseline.format_for_output(secrets)
 
         with self.get_baseline_file(formatter=label_secret) as f:
@@ -148,7 +149,7 @@ def label_secret(secrets):
             f.seek(0)
             data = json.loads(f.read())
 
-            assert data['results'][self.FILENAME][0]['is_secret']
+            assert data['results'][str(Path(self.FILENAME))][0]['is_secret']
 
     def test_maintains_slim_mode(self):
         with self.get_baseline_file(