From d0df05a571c85b6d2e70a3f280da9ba3c230a415 Mon Sep 17 00:00:00 2001 From: John-Paul Dakran Date: Wed, 2 Feb 2022 14:33:19 -0500 Subject: [PATCH 1/2] Bug Fix: Plugins Fail To Load on Python 3.8+ (#505) * Set fork multiprocessing as the upgrade to python 3.8 switched to spawn processing which does not share memory between child processes. * Pass settings object into child processes. Create the baseline json only once Co-authored-by: John-Paul Dakran --- detect_secrets/core/secrets_collection.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index 05f770a73..b477b9d99 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -11,6 +11,8 @@ from . import scan from .potential_secret import PotentialSecret +from detect_secrets.settings import configure_settings_from_baseline +from detect_secrets.settings import get_settings class PatchedFile: @@ -55,7 +57,13 @@ def scan_files(self, *filenames: str, num_processors: Optional[int] = None) -> N if not num_processors: num_processors = mp.cpu_count() - with mp.Pool(processes=num_processors) as p: + child_process_settings = get_settings().json() + + with mp.Pool( + processes=num_processors, + initializer=configure_settings_from_baseline, + initargs=(child_process_settings,), + ) as p: for secrets in p.imap_unordered( _scan_file_and_serialize, [os.path.join(self.root, filename) for filename in filenames], From 75a5844122aa93ece8a33237770cc6b95544e1a7 Mon Sep 17 00:00:00 2001 From: John-Paul Dakran Date: Wed, 2 Feb 2022 14:46:02 -0500 Subject: [PATCH 2/2] When checking the equality of secrets in a file, the key in the set should be the tuple of hash and type. This is because a single secret/hash can have multiple types (#507) --- detect_secrets/core/secrets_collection.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index b477b9d99..d3fc4dd56 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -249,8 +249,12 @@ def __eq__(self, other: Any, strict: bool = False) -> bool: return False for filename in self.files: - self_mapping = {secret.secret_hash: secret for secret in self[filename]} - other_mapping = {secret.secret_hash: secret for secret in other[filename]} + self_mapping = { + (secret.secret_hash, secret.type): secret for secret in self[filename] + } + other_mapping = { + (secret.secret_hash, secret.type): secret for secret in other[filename] + } # Since PotentialSecret is hashable, we compare their identities through this. if set(self_mapping.values()) != set(other_mapping.values()): @@ -260,7 +264,7 @@ def __eq__(self, other: Any, strict: bool = False) -> bool: continue for secretA in self_mapping.values(): - secretB = other_mapping[secretA.secret_hash] + secretB = other_mapping[(secretA.secret_hash, secretA.type)] valuesA = vars(secretA) valuesA.pop('secret_value')