From e4e0f94f3b8237f0994380d24b1a2c61d1f528c1 Mon Sep 17 00:00:00 2001 From: eddiez9 Date: Wed, 27 Jul 2022 09:52:19 +1000 Subject: [PATCH 1/3] update core bits to convert files to local filesystem --- detect_secrets/core/potential_secret.py | 3 ++- detect_secrets/core/secrets_collection.py | 7 ++++--- detect_secrets/util/path.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index eba034c79..be1cf8921 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -6,6 +6,7 @@ from ..util.color import AnsiColor from ..util.color import colorize +from ..util.path import convert_local_os_path class PotentialSecret: @@ -75,7 +76,7 @@ def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'Poten """Custom JSON decoder""" kwargs: Dict[str, Any] = { 'type': str(data['type']), - 'filename': str(data['filename']), + 'filename': convert_local_os_path(data['filename']), 'secret': 'will be replaced', } diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index d3fc4dd56..094a274ed 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -10,6 +10,7 @@ from typing import Tuple from . import scan +from ..util.path import convert_local_os_path from .potential_secret import PotentialSecret from detect_secrets.settings import configure_settings_from_baseline from detect_secrets.settings import get_settings @@ -40,7 +41,7 @@ def load_from_baseline(cls, baseline: Dict[str, Any]) -> 'SecretsCollection': for filename in baseline['results']: for item in baseline['results'][filename]: secret = PotentialSecret.load_secret_from_dict({'filename': filename, **item}) - output[filename].add(secret) + output[convert_local_os_path(filename)].add(secret) return output @@ -72,8 +73,8 @@ def scan_files(self, *filenames: str, num_processors: Optional[int] = None) -> N self[os.path.relpath(secret.filename, self.root)].add(secret) def scan_file(self, filename: str) -> None: - for secret in scan.scan_file(os.path.join(self.root, filename)): - self[filename].add(secret) + for secret in scan.scan_file(os.path.join(self.root, convert_local_os_path(filename))): + self[convert_local_os_path(filename)].add(secret) def scan_diff(self, diff: str) -> None: """ diff --git a/detect_secrets/util/path.py b/detect_secrets/util/path.py index 36eb15c64..acb744b81 100644 --- a/detect_secrets/util/path.py +++ b/detect_secrets/util/path.py @@ -16,3 +16,13 @@ def get_relative_path_if_in_cwd(path: str) -> Optional[str]: return filepath return None + + +def convert_local_os_path(path: str) -> str: + # Linux filesystem, replace \\ with / + if os.sep == '/': + path = path.replace('\\', '/') + return path + else: + path = path.replace('/', '\\') + return path From 24ca910f8533b36729e14d8756afeffc72142f09 Mon Sep 17 00:00:00 2001 From: eddiez9 Date: Wed, 27 Jul 2022 09:52:54 +1000 Subject: [PATCH 2/3] update test assertions to be os local path --- tests/audit/audit_test.py | 3 +- tests/core/baseline_test.py | 14 ++++++--- tests/core/secrets_collection_test.py | 44 ++++++++++++++------------- tests/pre_commit_hook_test.py | 7 +++-- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/tests/audit/audit_test.py b/tests/audit/audit_test.py index 46099c961..5de9d0636 100644 --- a/tests/audit/audit_test.py +++ b/tests/audit/audit_test.py @@ -1,5 +1,6 @@ import json import random +from pathlib import Path from typing import List from typing import Optional from unittest import mock @@ -135,7 +136,7 @@ def test_ensure_file_transformers_are_used(printer): run_logic(secrets, 'y') assert not m.called - line_number = list(secrets['test_data/config.env'])[0].line_number + line_number = list(secrets[str(Path('test_data/config.env'))])[0].line_number assert lines[line_number - 1] in printer.message diff --git a/tests/core/baseline_test.py b/tests/core/baseline_test.py index c8e24a8f3..d8237779c 100644 --- a/tests/core/baseline_test.py +++ b/tests/core/baseline_test.py @@ -1,4 +1,5 @@ import json +import os import subprocess import tempfile from pathlib import Path @@ -84,16 +85,21 @@ def test_scan_all_files(): def test_load_and_output(): with open('.secrets.baseline') as f: - data = json.loads(f.read()) + filedata = f.read() - secrets = baseline.load(data, filename='.secrets.baseline') + if os.sep == '\\': + # Replace Linux path seperators for Windows ones + filedata = filedata.replace('/', '\\\\') + + filedata_json = json.loads(filedata) + secrets = baseline.load(filedata_json, filename='.secrets.baseline') output = baseline.format_for_output(secrets) - for item in [data, output]: + for item in [filedata_json, output]: item.pop('generated_at') # We perform string matching because we want to ensure stable sorts. - assert json.dumps(output) == json.dumps(data) + assert json.dumps(output) == json.dumps(filedata_json) # We need to make sure that default values carry through, for future backwards compatibility. for plugin in output['plugins_used']: diff --git a/tests/core/secrets_collection_test.py b/tests/core/secrets_collection_test.py index 1f1fa3db8..8e7b9a6b7 100644 --- a/tests/core/secrets_collection_test.py +++ b/tests/core/secrets_collection_test.py @@ -1,3 +1,4 @@ +from pathlib import Path from unittest import mock import pytest @@ -43,7 +44,8 @@ def test_error_reading_file(mock_log_warning): ): SecretsCollection().scan_file('test_data/config.env') - assert 'Unable to open file: test_data/config.env' in mock_log_warning.warning_messages + file_warning = str(Path('test_data/config.env')) + assert 'Unable to open file: %s' % file_warning in mock_log_warning.warning_messages @staticmethod def test_line_based_success(): @@ -64,9 +66,9 @@ def test_line_based_success(): secrets = SecretsCollection() secrets.scan_file('test_data/each_secret.py') - secret = next(iter(secrets['test_data/each_secret.py'])) + secret = next(iter(secrets[str(Path('test_data/each_secret.py'))])) assert secret.secret_value.startswith('c2VjcmV0IG1lc') - assert len(secrets['test_data/each_secret.py']) == 1 + assert len(secrets[str(Path('test_data/each_secret.py'))]) == 1 @staticmethod def test_file_based_success_config(): @@ -80,11 +82,11 @@ def test_file_based_success_config(): secrets.scan_file('test_data/config.ini') assert [str(secret).splitlines()[1] for _, secret in secrets] == [ - 'Location: test_data/config.ini:2', - 'Location: test_data/config.ini:10', - 'Location: test_data/config.ini:21', - 'Location: test_data/config.ini:22', - 'Location: test_data/config.ini:32', + 'Location: %s:2' % str(Path('test_data/config.ini')), + 'Location: %s:10' % str(Path('test_data/config.ini')), + 'Location: %s:21' % str(Path('test_data/config.ini')), + 'Location: %s:22' % str(Path('test_data/config.ini')), + 'Location: %s:32' % str(Path('test_data/config.ini')), ] @staticmethod @@ -99,9 +101,9 @@ def test_file_based_success_yaml(): secrets.scan_file('test_data/config.yaml') assert [str(secret).splitlines()[1] for _, secret in secrets] == [ - 'Location: test_data/config.yaml:3', - 'Location: test_data/config.yaml:5', - 'Location: test_data/config.yaml:13', + 'Location: %s:3' % str(Path('test_data/config.yaml')), + 'Location: %s:5' % str(Path('test_data/config.yaml')), + 'Location: %s:13' % str(Path('test_data/config.yaml')), ] @staticmethod @@ -217,12 +219,12 @@ def test_deleted_secret(): secrets.scan_file('test_data/each_secret.py') results = SecretsCollection.load_from_baseline({'results': secrets.json()}) - results.data['test_data/each_secret.py'].pop() + results.data[str(Path('test_data/each_secret.py'))].pop() - original_size = len(secrets['test_data/each_secret.py']) + original_size = len(secrets[str(Path('test_data/each_secret.py'))]) secrets.trim(results) - assert len(secrets['test_data/each_secret.py']) < original_size + assert len(secrets[str(Path('test_data/each_secret.py'))]) < original_size @staticmethod def test_deleted_secret_file(): @@ -232,7 +234,7 @@ def test_deleted_secret_file(): secrets.trim(SecretsCollection()) assert secrets - secrets.trim(SecretsCollection(), filelist=['test_data/each_secret.py']) + secrets.trim(SecretsCollection(), filelist=[str(Path('test_data/each_secret.py'))]) assert not secrets @staticmethod @@ -288,7 +290,7 @@ def test_remove_non_existent_files(): secrets.scan_file('test_data/each_secret.py') assert bool(secrets) - secrets.data['does-not-exist'] = secrets.data.pop('test_data/each_secret.py') + secrets.data['does-not-exist'] = secrets.data.pop(str(Path('test_data/each_secret.py'))) secrets.trim() assert not bool(secrets) @@ -316,7 +318,7 @@ def test_bool(): secrets.scan_file('test_data/each_secret.py') assert secrets - secrets['test_data/each_secret.py'].clear() + secrets[str(Path('test_data/each_secret.py'))].clear() assert not secrets @@ -373,8 +375,8 @@ def test_basic(configure_plugins): assert secrets != baseline result = secrets - baseline - assert len(result['test_data/each_secret.py']) == 3 - assert len(secrets['test_data/each_secret.py']) == 5 + assert len(result[str(Path('test_data/each_secret.py'))]) == 3 + assert len(secrets[str(Path('test_data/each_secret.py'))]) == 5 @staticmethod def test_no_overlapping_files(configure_plugins): @@ -384,5 +386,5 @@ def test_no_overlapping_files(configure_plugins): secrets_a.scan_file('test_data/each_secret.py') secrets_b.scan_file('test_data/config.env') - assert (secrets_a - secrets_b).files == {'test_data/each_secret.py'} - assert (secrets_b - secrets_a).files == {'test_data/config.env'} + assert (secrets_a - secrets_b).files == {str(Path('test_data/each_secret.py'))} + assert (secrets_b - secrets_a).files == {str(Path('test_data/config.env'))} diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index c1d48a290..a7a595d72 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -3,6 +3,7 @@ import sys from contextlib import contextmanager from functools import partial +from pathlib import Path from typing import List from unittest import mock @@ -68,7 +69,7 @@ def test_baseline_filters_out_known_secrets(): ]) # Remove one arbitrary secret, so that it won't be the full set. - secrets.data['test_data/each_secret.py'].pop() + secrets.data[str(Path('test_data/each_secret.py'))].pop() with mock_named_temporary_file() as f: baseline.save_to_file(secrets, f.name) @@ -135,7 +136,7 @@ def test_success(self): def test_maintains_labelled_data(self): def label_secret(secrets): - list(secrets[self.FILENAME])[0].is_secret = True + list(secrets[str(Path(self.FILENAME))])[0].is_secret = True return baseline.format_for_output(secrets) with self.get_baseline_file(formatter=label_secret) as f: @@ -148,7 +149,7 @@ def label_secret(secrets): f.seek(0) data = json.loads(f.read()) - assert data['results'][self.FILENAME][0]['is_secret'] + assert data['results'][str(Path(self.FILENAME))][0]['is_secret'] def test_maintains_slim_mode(self): with self.get_baseline_file( From 505dc4b99de3b0c732638c9fa604b35a52bdd0b0 Mon Sep 17 00:00:00 2001 From: eddiez9 Date: Wed, 27 Jul 2022 10:21:14 +1000 Subject: [PATCH 3/3] add type cast --- detect_secrets/core/potential_secret.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index be1cf8921..840d3291c 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -76,7 +76,7 @@ def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'Poten """Custom JSON decoder""" kwargs: Dict[str, Any] = { 'type': str(data['type']), - 'filename': convert_local_os_path(data['filename']), + 'filename': convert_local_os_path(str(data['filename'])), 'secret': 'will be replaced', }