feat: add ignore_known_secrets option to secret scan

.gitguardian.yaml

@@ -20,6 +20,8 @@ secret:
       name:
     - match: 1a56aa9f3d76450ec4731ba7ae247176dfb460dca2989c40ed634eb9a2c1c262
       name: Generic High Entropy Secret - tests/output/snapshots/snap_test_text_output.py
+    - match: 6a1d11d5bbbb48f044bedfc2db5749e7fde3ad7e3f6a79601db034478a95f88a
+      name: Facebook Access Token - tests/output/snapshots/snap_test_text_output.py
     - match: 793865a3729a2c5d38ef874a270accf407cd993af32acefb4e13fc1ea242e9cb
       name: Generic High Entropy Secret - tests/cassettes/test_scan_file_secret-False.yaml
     - match: 56c126cef75e3d17c3de32dac60bab688ecc384a054c2c85b688c1dd7ac4eefd

.github/workflows/main.yml

@@ -27,7 +27,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install pipenv pre-commit
-          pipenv install --dev --skip-lock
+          pipenv install --dev
 
       - uses: actions/cache@v3
         with:

Pipfile

@@ -13,7 +13,7 @@ click = ">=8.1,<8.2"
 marshmallow = ">=3.18.0,<3.19.0"
 marshmallow-dataclass = ">=8.5.8,<8.6.0"
 oauthlib = ">=3.2.1,<3.3.0"
-pygitguardian = ">=1.3.6,<1.4.0"
+pygitguardian = {git = "https://github.com/GitGuardian/py-gitguardian.git", editable = true}
 python-dotenv = ">=0.21.0,<0.22.0"
 pyyaml = ">=6.0,<6.1"
 rich = ">=12.5.1,<12.6.0"

ggshield/cmd/secret/scan/__init__.py

@@ -90,6 +90,12 @@
     is_flag=True,
     hidden=True,
 )
+@click.option(
+    "--ignore-known-secrets",
+    is_flag=True,
+    default=None,
+    help="Ignore",
+)
 @click.pass_context
 def scan_group(
     ctx: click.Context,
@@ -102,6 +108,7 @@ def scan_group(
     banlist_detector: Optional[List[str]] = None,
     exclude: Optional[List[str]] = None,
     ignore_default_excludes: bool = False,
+    ignore_known_secrets: bool = False,
 ) -> int:
     """Commands to scan various contents."""
     return scan_group_impl(
@@ -113,6 +120,7 @@ def scan_group(
         output,
         banlist_detector,
         exclude,
+        ignore_known_secrets,
     )
 
 
@@ -125,6 +133,7 @@ def scan_group_impl(
     output: Optional[str],
     banlist_detector: Optional[List[str]] = None,
     exclude: Optional[List[str]] = None,
+    ignore_known_secrets: bool = False,
 ) -> int:
     """Implementation for scan_group(). Must be a separate function so that its code can
     be reused from the deprecated `cmd.scan` package."""
@@ -152,6 +161,9 @@ def scan_group_impl(
     if banlist_detector:
         config.secret.ignored_detectors.update(banlist_detector)
 
+    if ignore_known_secrets:
+        config.ignore_known_secrets = ignore_known_secrets
+
     max_commits = get_max_commits_for_hook()
     if max_commits:
         config.max_commits_for_hook = max_commits

ggshield/cmd/secret/scan/archive.py

@@ -53,6 +53,7 @@ def archive_cmd(ctx: click.Context, path: str) -> int:  # pragma: no cover
                 scan_context=scan_context,
                 ignored_matches=config.secret.ignored_matches,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
             results = scanner.scan(
                 files.files,

ggshield/cmd/secret/scan/docker.py

@@ -51,6 +51,7 @@ def docker_name_cmd(ctx: click.Context, name: str, docker_timeout: int) -> int:
                 scan_context=scan_context,
                 matches_ignore=config.secret.ignored_matches,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
 
             return output_handler.process_scan(scan)

ggshield/cmd/secret/scan/dockerarchive.py

@@ -38,6 +38,7 @@ def docker_archive_cmd(
             matches_ignore=config.secret.ignored_matches,
             scan_context=scan_context,
             ignored_detectors=config.secret.ignored_detectors,
+            ignore_known_secrets=config.ignore_known_secrets,
         )
 
         return output_handler.process_scan(scan)

ggshield/cmd/secret/scan/docset.py

@@ -68,6 +68,7 @@ def docset_cmd(ctx: click.Context, files: List[TextIO]) -> int:  # pragma: no co
                 ignored_matches=config.secret.ignored_matches,
                 scan_context=scan_context,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
 
             scans = create_scans_from_docset_files(

ggshield/cmd/secret/scan/path.py

@@ -53,6 +53,7 @@ def path_cmd(
                 ignored_matches=config.secret.ignored_matches,
                 scan_context=scan_context,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
             results = scanner.scan(
                 files.files,

ggshield/cmd/secret/scan/precommit.py

@@ -50,6 +50,7 @@ def precommit_cmd(
             scan_context=scan_context,
             ignored_matches=config.secret.ignored_matches,
             ignored_detectors=config.secret.ignored_detectors,
+            ignore_known_secrets=config.ignore_known_secrets,
         )
         results = scanner.scan(commit.files)
 

ggshield/cmd/secret/scan/prepush.py

@@ -120,6 +120,7 @@ def prepush_cmd(ctx: click.Context, prepush_args: List[str]) -> int:
             matches_ignore=config.secret.ignored_matches,
             scan_context=scan_context,
             ignored_detectors=config.secret.ignored_detectors,
+            ignore_known_secrets=config.ignore_known_secrets,
         )
         if return_code:
             click.echo(

ggshield/cmd/secret/scan/prereceive.py

@@ -201,6 +201,7 @@ def prereceive_cmd(ctx: click.Context, web: bool, prereceive_args: List[str]) ->
                 matches_ignore=config.secret.ignored_matches,
                 scan_context=scan_context,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
             if return_code:
                 click.echo(

ggshield/cmd/secret/scan/pypi.py

@@ -115,6 +115,7 @@ def pypi_cmd(ctx: click.Context, package_name: str) -> int:  # pragma: no cover
                 ignored_matches=config.secret.ignored_matches,
                 scan_context=scan_context,
                 ignored_detectors=config.secret.ignored_detectors,
+                ignore_known_secrets=config.ignore_known_secrets,
             )
             results = scanner.scan(
                 files.files,

ggshield/cmd/secret/scan/range.py

@@ -38,6 +38,7 @@ def range_cmd(ctx: click.Context, commit_range: str) -> int:  # pragma: no cover
             matches_ignore=config.secret.ignored_matches,
             scan_context=scan_context,
             ignored_detectors=config.secret.ignored_detectors,
+            ignore_known_secrets=config.ignore_known_secrets,
         )
     except Exception as error:
         return handle_exception(error, config.verbose)

ggshield/core/config/user_config.py

@@ -91,6 +91,7 @@ class UserConfig(FilteredConfig):
     allow_self_signed: bool = False
     max_commits_for_hook: int = 50
     secret: SecretConfig = field(default_factory=SecretConfig)
+    ignore_known_secrets: bool = False
     debug: bool = False
 
     # If we hit any deprecated syntax when loading a configuration file, we do not

ggshield/core/text_utils.py

@@ -37,12 +37,6 @@
     "ignore_sha": {"fg": "bright_yellow", "bold": True},
 }
 
-_DISPLAY_NAME_FOR_POLICY = {"secrets detection": "Secret detected"}
-
-
-def get_display_name_for_policy(policy: str) -> str:
-    return _DISPLAY_NAME_FOR_POLICY.get(policy.lower(), policy)
-
 
 class LineCategory(Enum):
     addition = auto()

ggshield/output/json/json_output_handler.py

@@ -117,6 +117,9 @@ def flattened_policy_break(
         if policy_breaks[0].validity:
             flattened_dict["validity"] = policy_breaks[0].validity
 
+        if policy_breaks[0].known_secret:
+            flattened_dict["known_secret"] = policy_breaks[0].known_secret
+
         for policy_break in policy_breaks:
             matches = JSONOutputHandler.make_matches(
                 policy_break.matches, lines, is_patch

ggshield/output/json/schemas.py

@@ -77,6 +77,7 @@ class FlattenedPolicyBreak(BaseSchema):
     validity = fields.String(required=False, allow_none=True)
     ignore_sha = fields.String(required=True)
     total_occurrences = fields.Integer(required=True)
+    known_secret = fields.Bool(required=False)
 
 
 class JSONResultSchema(BaseSchema):

ggshield/output/output_handler.py

@@ -49,8 +49,8 @@ def _process_scan_impl(self, scan: ScanCollection) -> str:
 
     @staticmethod
     def _get_exit_code(scan: ScanCollection) -> int:
-        if scan.has_results or scan.has_iac_result:
+        if scan.has_iac_result:
             return 1
-        if scan.scans and any(x.has_results for x in scan.scans):
+        if scan.has_new_secrets:
             return 1
         return 0