feat(perf-issues): Add CLI script for testing detector output

src/sentry/runner/__init__.py

@@ -70,6 +70,8 @@ def cli(ctx, config):
         "sentry.runner.commands.upgrade.upgrade",
         "sentry.runner.commands.permissions.permissions",
         "sentry.runner.commands.devservices.devservices",
+        "sentry.runner.commands.performance.performance",
+        "sentry.runner.commands.performance.detect",
     ),
 ):
     cli.add_command(cmd)

src/sentry/runner/commands/performance.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+from inspect import isclass
+
+import click
+from django.template.defaultfilters import pluralize
+
+from sentry.runner.decorators import configuration
+from sentry.utils import json
+
+
+@click.group()
+def performance() -> None:
+    """
+    Performance utilities
+    """
+
+
+@performance.command()
+@click.argument("filename", type=click.Path(exists=True))
+@click.option("-v", "--verbose", count=True)
+@configuration
+def detect(filename, verbose):
+    """
+    Runs performance problem detection on event data in the supplied filename
+    using default detector settings with every detector. Filename should be a
+    path to a JSON event data file.
+    """
+    from sentry.utils.performance_issues import performance_detection
+
+    detector_classes = [
+        cls
+        for _, cls in performance_detection.__dict__.items()
+        if isclass(cls)
+        and issubclass(cls, performance_detection.PerformanceDetector)
+        and cls != performance_detection.PerformanceDetector
+    ]
+
+    settings = performance_detection.get_detection_settings()
+
+    with open(filename) as file:
+        data = json.loads(file.read())
+        click.echo(f"Event ID: {data['event_id']}")
+
+        detectors = [cls(settings, data) for cls in detector_classes]
+
+        for detector in detectors:
+            click.echo(f"Detecting using {detector.__class__.__name__}")
+            performance_detection.run_detector_on_data(detector, data)
+
+            if len(detector.stored_problems) == 0:
+                click.echo("No problems detected")
+            else:
+                click.echo(
+                    f"Found {len(detector.stored_problems)} {pluralize(len(detector.stored_problems), 'problem,problems')}"
+                )
+
+            if verbose > 0:
+                for problem in detector.stored_problems.values():
+                    try:
+                        click.echo(problem.to_dict())
+                    except AttributeError:
+                        click.echo(problem)
+
+            click.echo("\n")

src/sentry/utils/performance_issues/performance_detection.py

@@ -193,13 +193,22 @@ def detect_performance_problems(data: Event) -> List[PerformanceProblem]:
 # Gets the thresholds to perform performance detection.
 # Duration thresholds are in milliseconds.
 # Allowed span ops are allowed span prefixes. (eg. 'http' would work for a span with 'http.client' as its op)
-def get_detection_settings(project_id: str):
-    default_project_settings = projectoptions.get_well_known_default(
-        "sentry:performance_issue_settings",
-        project=project_id,
+def get_detection_settings(project_id: Optional[str] = None):
+    default_project_settings = (
+        projectoptions.get_well_known_default(
+            "sentry:performance_issue_settings",
+            project=project_id,
+        )
+        if project_id
+        else {}
     )
-    project_settings = ProjectOption.objects.get_value(
-        project_id, "sentry:performance_issue_settings", default_project_settings
+
+    project_settings = (
+        ProjectOption.objects.get_value(
+            project_id, "sentry:performance_issue_settings", default_project_settings
+        )
+        if project_id
+        else {}
     )
 
     use_project_option_settings = default_project_settings != project_settings
@@ -284,7 +293,6 @@ def get_detection_settings(project_id: str):
 
 def _detect_performance_problems(data: Event, sdk_span: Any) -> List[PerformanceProblem]:
     event_id = data.get("event_id", None)
-    spans = data.get("spans", [])
     project_id = data.get("project")
 
     detection_settings = get_detection_settings(project_id)
@@ -304,11 +312,8 @@ def _detect_performance_problems(data: Event, sdk_span: Any) -> List[Performance
         ),
     }
 
-    for span in spans:
-        for _, detector in detectors.items():
-            detector.visit_span(span)
     for _, detector in detectors.items():
-        detector.on_complete()
+        run_detector_on_data(detector, data)
 
     # Metrics reporting only for detection, not created issues.
     report_metrics_for_detectors(data, event_id, detectors, sdk_span)
@@ -346,6 +351,14 @@ def _detect_performance_problems(data: Event, sdk_span: Any) -> List[Performance
     return list(unique_performance_problems)
 
 
+def run_detector_on_data(detector, data):
+    spans = data.get("spans", [])
+    for span in spans:
+        detector.visit_span(span)
+
+    detector.on_complete()
+
+
 # Uses options and flags to determine which orgs and which detectors automatically create performance issues.
 def get_allowed_issue_creation_detectors(project_id: str):
     project = Project.objects.get_from_cache(id=project_id)