Add overrides via .semgrepconfig.yml

README.md

@@ -112,6 +112,46 @@ docker run -v $(pwd):/src --workdir /src returntocorp/semgrep-agent:v1 semgrep-a
 > which can be changed to any value
 > that `semgrep` itself understands.
 
+## EXPERIMENTAL: `.semgrepconfig.yml` overrides
+
+You can add a `.semgrepconfig.yml` that looks like this:
+
+```yaml
+overrides:
+  - if.path: "tests/*" # the first two lines in this example are "conditions"
+    if.rule_id: "secrets.*aws*"
+    mute: true # this last line in this example is an "action"
+
+  - if.policy_slug: "*important*"
+    if.severity_in: ["ERROR"]
+    unmute: true # report issues even if they were muted with # nosemgrep
+    set_severity: WARNING # but lower their severity a bit
+```
+
+Every finding will be checked against these override definitions one by one.
+The override definitions will run in the same order you have them in your config file.
+
+An override's actions are applied when all `if.` conditions are true. If an override applies, it'll take actions as described by keys not starting with `if.*`.
+
+### Available conditions
+
+| key              | example value         | `True` if the finding's…                        |
+| ---------------- | --------------------- | ----------------------------------------------- |
+| `if.path`        | `"tests/*"`           | path matches the given glob                     |
+| `if.rule_id`     | `"secrets.*aws*"`     | rule ID matches the given glob                  |
+| `if.ruleset_id`  | `"secrets"`           | rule is from a ruleset matching the given glob  |
+| `if.finding_id`  | `"1fd8aac00"`         | finding's `syntactic_id` starts with this value |
+| `if.policy_slug` | `"security*"`         | rule is from a policy matching the given glob   |
+| `if.severity_in` | `["INFO", "WARNING"]` | rule's severity is in the given list            |
+
+### Available actions
+
+| key            | example value | description                                     |
+| -------------- | ------------- | ----------------------------------------------- |
+| `mute`         | `true`        | acts as if the line had a `# nosemgrep` comment |
+| `unmute`       | `true`        | ignores any `# nosemgrep` comments              |
+| `set_severity` | `"INFO"`      | changes the reported severity of the issue      |
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md)

src/semgrep_agent/findings.py

@@ -94,7 +94,7 @@ def semgrep_severity_to_int(severity: str) -> int:
     def from_semgrep_result(
         cls,
         result: Dict[str, Any],
-        committed_datetime: Optional[datetime],
+        committed_datetime: Optional[datetime] = None,
     ) -> "Finding":
         return cls(
             check_id=result["check_id"],
@@ -116,7 +116,7 @@ def to_dict(
     ) -> Mapping[str, Any]:
         d = attr.asdict(self)
         d = {k: v for k, v in d.items() if v is not None and k not in omit}
-        d["syntactic_id"] = self.syntactic_identifier_str()
+        d["finding_id"] = d["syntactic_id"] = self.syntactic_identifier_str()
         d["commit_date"] = d["commit_date"].isoformat()
         d["is_blocking"] = self.is_blocking()
         return d

src/semgrep_agent/postprocessing.py

@@ -0,0 +1,134 @@
+from copy import deepcopy
+from fnmatch import fnmatch
+from functools import partial
+from pathlib import Path
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from ruamel.yaml import YAML  # type: ignore
+
+from semgrep_agent.findings import Finding
+
+yaml = YAML(typ="rt")
+CONFIG_PATH = Path.cwd() / ".semgrepconfig.yml"
+
+OVERRIDE_CONDITIONS: Dict[str, Callable] = {}
+OVERRIDE_ACTIONS: Dict[str, Callable] = {}
+
+
+def register_function(
+    function_mapping: Dict[str, Callable], override_key: str
+) -> Callable[[Callable], Callable]:
+    def wrapper(function: Callable) -> Callable:
+        function_mapping[override_key] = function
+        return function
+
+    return wrapper
+
+
+register_condition = partial(register_function, OVERRIDE_CONDITIONS)
+register_action = partial(register_function, OVERRIDE_ACTIONS)
+
+
+@register_condition("if.path")
+def if_path(config_value: str, result: Dict[str, Any]) -> bool:
+    return fnmatch(result.get("path", ""), config_value)
+
+
+@register_condition("if.rule_id")
+def if_rule_id(config_value: str, result: Dict[str, Any]) -> bool:
+    return fnmatch(result.get("check_id", ""), config_value)
+
+
+@register_condition("if.ruleset_id")
+def if_ruleset_id(config_value: str, result: Dict[str, Any]) -> bool:
+    return fnmatch(result.get("metadata", {}).get("semgrep.ruleset", ""), config_value)
+
+
+@register_condition("if.policy_slug")
+def if_policy_slug(config_value: str, result: Dict[str, Any]) -> bool:
+    return fnmatch(
+        result.get("metadata", {}).get("semgrep.policy", {}).get("slug", ""),
+        config_value,
+    )
+
+
+@register_condition("if.severity_in")
+def if_severity_in(config_value: List[str], result: Dict[str, Any]) -> bool:
+    return result.get("extra", {}).get("severity", "") in config_value
+
+
+@register_condition("if.finding_id")
+def if_finding_id(config_value: str, result: Dict[str, Any]) -> bool:
+    return (
+        Finding.from_semgrep_result(result)
+        .syntactic_identifier_str()
+        .startswith(config_value)
+    )
+
+
+@register_action("mute")
+def mute(config_value: bool, result: Dict[str, Any]) -> Dict[str, Any]:
+    if not config_value:
+        return result
+
+    result.setdefault("extra", {})
+    result["extra"]["is_ignored"] = True
+
+    return result
+
+
+@register_action("unmute")
+def unmute(config_value: bool, result: Dict[str, Any]) -> Dict[str, Any]:
+    if not config_value:
+        return result
+
+    result.setdefault("extra", {})
+    result["extra"]["is_ignored"] = False
+
+    return result
+
+
+@register_action("set_severity")
+def set_severity(config_value: str, result: Dict[str, Any]) -> Dict[str, Any]:
+    result.setdefault("extra", {})
+    result["extra"]["severity"] = config_value
+    return result
+
+
+def load_config() -> Optional[Dict[str, Any]]:
+    if not CONFIG_PATH.exists():
+        return None
+    config = yaml.load(CONFIG_PATH.read_text())
+    return cast(Dict[str, Any], config)
+
+
+def update_result(result: Dict, overrides: List) -> Dict[str, Any]:
+    for override in overrides:
+        if all(
+            OVERRIDE_CONDITIONS[override_key](override_value, result)
+            for override_key, override_value in override.items()
+            if override_key in OVERRIDE_CONDITIONS
+        ):
+            for override_key, override_value in override.items():
+                if override_key in OVERRIDE_ACTIONS:
+                    result = OVERRIDE_ACTIONS[override_key](override_value, result)
+
+    return result
+
+
+def postprocess(results: List) -> List:
+    config = load_config()
+    if not config:
+        return results
+
+    new_results = deepcopy(results)
+
+    for result in new_results:
+        result = update_result(result, config["overrides"])
+
+    return new_results

src/semgrep_agent/semgrep.py

@@ -26,6 +26,7 @@
 from semgrep_agent.exc import ActionFailure
 from semgrep_agent.findings import Finding
 from semgrep_agent.findings import FindingSets
+from semgrep_agent.postprocessing import postprocess
 from semgrep_agent.targets import TargetFileManager
 from semgrep_agent.utils import debug_echo
 from semgrep_agent.utils import debug_file_descriptor
@@ -286,6 +287,8 @@ def invoke_semgrep(
                 output["results"].extend(parsed_output["results"])
                 output["errors"].extend(parsed_output["errors"])
 
+    output["results"] = postprocess(output["results"])
+
     return output
 
 

tests/acceptance/postprocess-generic/base-log.err

@@ -0,0 +1 @@
+

tests/acceptance/postprocess-generic/base-log.out

@@ -0,0 +1,8 @@
+commit aaaaaaa
+Author: Test <test@r2c.dev>
+Date:   YYYY-MM-DD
+
+    base
+
+ foo.py | 1 +
+ 1 file changed, 1 insertion(+)

tests/acceptance/postprocess-generic/commands.yaml

@@ -0,0 +1,93 @@
+steps:
+  - include: generic.yaml
+  # Tests .semgrepconfig.yml overrides
+  - name: new add
+    description: create a new file with findings
+    command:
+      - bash
+      - -c
+      - echo "2 == 2" > bar.py && git add bar.py && git commit -m 'add finding'
+
+  - name: no override
+    description: create an empty .semgrepconfig.yml
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: []" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with no override
+    description: run agent on a single new file with no override
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci
+    returncode: 1
+    expected_out: no-override.out
+    expected_err: no-override.err
+
+  - name: path override
+    description: create a .semgrepconfig.yml overriding by path
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: [{if.path: '*ba*.py*', mute: true}]" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with path override
+    description: run agent on a single new file with path override
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci
+    returncode: 0
+    expected_out: if-path-override.out
+    expected_err: if-path-override.err
+
+  - name: rule ID override
+    description: create a .semgrepconfig.yml overriding by rule ID
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: [{if.rule_id: '*eqeq*', mute: true}]" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with rule id override
+    description: run agent on a single new file with rule id override
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci
+    returncode: 0
+    expected_out: if-rule-id-override.out
+    expected_err: if-rule-id-override.err
+
+  - name: finding ID override
+    description: create a .semgrepconfig.yml overriding by finding ID
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: [{if.finding_id: 'a179ed11', mute: true}]" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with finding id override
+    description: run agent on a single new file with finding id override
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci
+    returncode: 0
+    expected_out: if-finding-id-override.out
+    expected_err: if-finding-id-override.err
+
+  - name: severity override
+    description: create a .semgrepconfig.yml overriding by severity
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: [{if.severity_in: [ERROR], mute: true}]" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with severity override
+    description: run agent on a single new file with severity override
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci
+    returncode: 0
+    expected_out: if-severity-override.out
+    expected_err: if-severity-override.err
+
+  - name: override severity
+    description: create a .semgrepconfig.yml overriding the severity
+    command:
+      - bash
+      - -c
+      - >-
+        echo "overrides: [{if.path: '*', set_severity: INFO}]" > .semgrepconfig.yml && git add .semgrepconfig.yml && git commit --amend --no-edit
+  - name: agent with overriding the severity
+    description: run agent on a single new file with overriding the severity
+    command: semgrep-agent --baseline-ref HEAD~1 --config p/r2c-ci --json
+    returncode: 1
+    expected_out: severity-action-override-json.out
+    expected_err: severity-action-override-json.err

tests/acceptance/postprocess-generic/if-finding-id-override.err

@@ -0,0 +1,15 @@
+| versions          - semgrep x.y.z on Python x.y.z
+| environment       - running in environment git, triggering event is 'unknown'
+| manage            - not logged in
+=== setting up agent configuration
+| using semgrep rules from https://semgrep.dev/c/p/r2c-ci
+| using default path ignore rules of common test and dependency directories
+| reporting findings introduced by these commits:
+|   * aaaaaaa add finding
+| looking at 2 changed paths
+| found 2 files in the paths to be scanned
+=== looking for current issues in 2 files
+| No current issues found
+| 1 ignored issue found
+=== not looking at pre-existing issues since there are no current issues
+=== exiting with success status

tests/acceptance/postprocess-generic/if-finding-id-override.out

@@ -0,0 +1 @@
+

tests/acceptance/postprocess-generic/if-path-override.err

@@ -0,0 +1,15 @@
+| versions          - semgrep x.y.z on Python x.y.z
+| environment       - running in environment git, triggering event is 'unknown'
+| manage            - not logged in
+=== setting up agent configuration
+| using semgrep rules from https://semgrep.dev/c/p/r2c-ci
+| using default path ignore rules of common test and dependency directories
+| reporting findings introduced by these commits:
+|   * aaaaaaa add finding
+| looking at 2 changed paths
+| found 2 files in the paths to be scanned
+=== looking for current issues in 2 files
+| No current issues found
+| 1 ignored issue found
+=== not looking at pre-existing issues since there are no current issues
+=== exiting with success status

tests/acceptance/postprocess-generic/if-path-override.out

@@ -0,0 +1 @@
+

tests/acceptance/postprocess-generic/if-rule-id-override.err

@@ -0,0 +1,15 @@
+| versions          - semgrep x.y.z on Python x.y.z
+| environment       - running in environment git, triggering event is 'unknown'
+| manage            - not logged in
+=== setting up agent configuration
+| using semgrep rules from https://semgrep.dev/c/p/r2c-ci
+| using default path ignore rules of common test and dependency directories
+| reporting findings introduced by these commits:
+|   * aaaaaaa add finding
+| looking at 2 changed paths
+| found 2 files in the paths to be scanned
+=== looking for current issues in 2 files
+| No current issues found
+| 1 ignored issue found
+=== not looking at pre-existing issues since there are no current issues
+=== exiting with success status

tests/acceptance/postprocess-generic/if-rule-id-override.out

@@ -0,0 +1 @@
+

tests/acceptance/postprocess-generic/if-severity-override.err

@@ -0,0 +1,15 @@
+| versions          - semgrep x.y.z on Python x.y.z
+| environment       - running in environment git, triggering event is 'unknown'
+| manage            - not logged in
+=== setting up agent configuration
+| using semgrep rules from https://semgrep.dev/c/p/r2c-ci
+| using default path ignore rules of common test and dependency directories
+| reporting findings introduced by these commits:
+|   * aaaaaaa add finding
+| looking at 2 changed paths
+| found 2 files in the paths to be scanned
+=== looking for current issues in 2 files
+| No current issues found
+| 1 ignored issue found
+=== not looking at pre-existing issues since there are no current issues
+=== exiting with success status

tests/acceptance/postprocess-generic/if-severity-override.out

@@ -0,0 +1 @@
+