feat(code-mappings): Add new task to find projects with missing code mappings

mypy.ini

@@ -113,6 +113,7 @@ files = fixtures/mypy-stubs,
         src/sentry/tasks/store.py,
         src/sentry/tasks/symbolication.py,
         src/sentry/tasks/update_user_reports.py,
+        src/sentry/tasks/derive_code_mappings.py,
         src/sentry/testutils/modelmanifest.py,
         src/sentry/testutils/silo.py,
         src/sentry/types/region.py,

src/sentry/conf/server.py

@@ -684,6 +684,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME():
     Queue("replays.delete_replay", routing_key="replays.delete_replay"),
     Queue("counters-0", routing_key="counters-0"),
     Queue("triggers-0", routing_key="triggers-0"),
+    Queue("derive_code_mappings", routing_key="derive_code_mappings"),
 ]
 
 for queue in CELERY_QUEUES:

src/sentry/tasks/derive_code_mappings.py

@@ -0,0 +1,100 @@
+import logging
+from datetime import timedelta
+from typing import Any, List, Mapping, Optional, Set, Tuple
+
+from django.utils import timezone
+
+from sentry.db.models.fields.node import NodeData
+from sentry.models import Project
+from sentry.models.group import Group
+from sentry.models.organization import Organization, OrganizationStatus
+from sentry.tasks.base import instrumented_task
+from sentry.utils.safe import get_path
+
+ACTIVE_PROJECT_THRESHOLD = timedelta(days=7)
+GROUP_ANALYSIS_RANGE = timedelta(days=14)
+
+logger = logging.getLogger("sentry.tasks.derive_code_mappings")
+
+
+@instrumented_task(  # type: ignore
+    name="sentry.tasks.derive_code_mappings.identify_stacktrace_paths",
+    queue="derive_code_mappings",
+    max_retries=0,  # if we don't backfill it this time, we'll get it the next time
+)
+def identify_stacktrace_paths(
+    organizations: Optional[List[Organization]] = None,
+) -> Mapping[str, Mapping[str, List[str]]]:
+    """
+    Generate a map of projects to stacktrace paths for specified organizations,
+    or all active organizations if unspecified.
+
+    This filters out projects have not had an event in the last 7 days or have
+    non-python files in the stacktrace.
+    """
+    if organizations is None:
+        organizations = Organization.objects.filter(status=OrganizationStatus.ACTIVE)
+
+    filename_maps = {}
+    for org in organizations:
+        projects = Project.objects.filter(organization=org, first_event__isnull=False)
+        projects = [
+            project
+            for project in projects
+            if Group.objects.filter(
+                project=project,
+                last_seen__gte=timezone.now() - ACTIVE_PROJECT_THRESHOLD,
+            ).exists()
+        ]
+
+        project_file_map = {project.slug: get_all_stacktrace_paths(project) for project in projects}
+        filename_maps[org.slug] = project_file_map
+    return filename_maps
+
+
+def get_all_stacktrace_paths(project: Project) -> List[str]:
+    groups = Group.objects.filter(
+        project=project, last_seen__gte=timezone.now() - GROUP_ANALYSIS_RANGE
+    )
+
+    all_stacktrace_paths = set()
+    for group in groups:
+        event = group.get_latest_event()
+        is_python_stacktrace, stacktrace_paths = get_stacktrace_paths(event.data)
+        if not is_python_stacktrace:
+            return []
+        all_stacktrace_paths.update(stacktrace_paths)
+
+    return list(all_stacktrace_paths)
+
+
+def get_stacktrace_paths(data: NodeData) -> Tuple[bool, Set[str]]:
+    """
+    Get the stacktrace_paths from the stacktrace for the latest event for an issue.
+    """
+    stacktraces = get_stacktrace(data)
+    stacktrace_paths = set()
+    for stacktrace in stacktraces:
+        try:
+            paths = [frame["filename"] for frame in stacktrace["frames"]]
+            if len(paths) == 0:
+                continue
+            if paths[0].endswith(".py"):
+                stacktrace_paths.update(paths)
+            else:
+                return False, set()  # (is_python, stacktrace_paths)
+        except Exception:
+            logger.exception("Error getting filenames for project {project.slug}")
+    return True, stacktrace_paths  # (is_python, stacktrace_paths)
+
+
+def get_stacktrace(data: NodeData) -> List[Mapping[str, Any]]:
+    exceptions = get_path(data, "exception", "values", filter=True)
+    if exceptions:
+        return [e["stacktrace"] for e in exceptions if get_path(e, "stacktrace", "frames")]
+
+    stacktrace = data.get("stacktrace")
+    if stacktrace and stacktrace.get("frames"):
+        return [stacktrace]
+
+    return []

tests/sentry/tasks/test_derive_code_mappings.py

@@ -0,0 +1,145 @@
+from copy import deepcopy
+
+from sentry.models.organization import OrganizationStatus
+from sentry.tasks.derive_code_mappings import identify_stacktrace_paths
+from sentry.testutils import TestCase
+from sentry.testutils.helpers.datetime import before_now, iso_format
+
+
+class TestCommitContext(TestCase):
+    def setUp(self):
+        self.organization = self.create_organization(status=OrganizationStatus.ACTIVE)
+        self.project = self.create_project(organization=self.organization)
+        self.test_data_1 = {
+            "message": "Kaboom!",
+            "platform": "python",
+            "timestamp": iso_format(before_now(days=1)),
+            "stacktrace": {
+                "frames": [
+                    {
+                        "function": "handle_set_commits",
+                        "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                        "module": "sentry.tasks",
+                        "in_app": False,
+                        "lineno": 30,
+                        "filename": "sentry/tasks.py",
+                    },
+                    {
+                        "function": "set_commits",
+                        "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                        "module": "sentry.models.release",
+                        "in_app": True,
+                        "lineno": 39,
+                        "filename": "sentry/models/release.py",
+                    },
+                ]
+            },
+            "fingerprint": ["put-me-in-the-control-group"],
+        }
+        self.test_data_2 = deepcopy(self.test_data_1)
+        self.test_data_2["stacktrace"]["frames"][0]["filename"] = "sentry/test_file.py"
+        self.test_data_2["stacktrace"]["frames"][1]["filename"] = "sentry/models/test_file.py"
+        self.test_data_2["fingerprint"] = ["new-group"]
+        self.test_data_2["timestamp"] = iso_format(before_now(days=2))
+
+    def test_finds_stacktrace_paths_single_project(self):
+        self.store_event(data=self.test_data_1, project_id=self.project.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths([self.organization])
+        assert self.organization.slug in mapping
+
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+
+    def test_finds_stacktrace_paths_multiple_projects(self):
+        project_2 = self.create_project(organization=self.organization)
+        self.store_event(data=self.test_data_1, project_id=self.project.id)
+        self.store_event(data=self.test_data_2, project_id=project_2.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths([self.organization])
+        assert self.organization.slug in mapping
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+        assert project_2.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[project_2.slug]) == [
+            "sentry/models/test_file.py",
+            "sentry/test_file.py",
+        ]
+
+    def test_finds_stacktrace_paths_multiple_orgs(self):
+        new_org = self.create_organization()
+        new_project = self.create_project(organization=new_org)
+        self.store_event(self.test_data_1, project_id=self.project.id)
+        self.store_event(data=self.test_data_2, project_id=new_project.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths([self.organization, new_org])
+        assert self.organization.slug in mapping
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+        assert new_org.slug in mapping
+        stacktrace_paths = mapping[new_org.slug]
+        assert new_project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[new_project.slug]) == [
+            "sentry/models/test_file.py",
+            "sentry/test_file.py",
+        ]
+
+    def test_skips_stale_projects(self):
+        stale_event = deepcopy(self.test_data_1)
+        stale_event["timestamp"] = iso_format(before_now(days=8))
+        self.store_event(data=stale_event, project_id=self.project.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths()
+        assert self.organization.slug in mapping
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug not in stacktrace_paths
+
+    def test_skips_outdated_events(self):
+        stale_event = deepcopy(self.test_data_2)
+        stale_event["timestamp"] = iso_format(before_now(days=16))
+        self.store_event(data=self.test_data_1, project_id=self.project.id)
+        self.store_event(data=stale_event, project_id=self.project.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths([self.organization])
+        assert self.organization.slug in mapping
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+
+    def test_handle_duplicate_filenames_in_a_project(self):
+        self.store_event(data=self.test_data_1, project_id=self.project.id)
+        duplicate_event = deepcopy(self.test_data_2)
+        duplicate_event["stacktrace"]["frames"].append(self.test_data_1["stacktrace"]["frames"][0])
+        self.store_event(data=duplicate_event, project_id=self.project.id)
+
+        with self.tasks():
+            mapping = identify_stacktrace_paths([self.organization])
+        assert self.organization.slug in mapping
+        stacktrace_paths = mapping[self.organization.slug]
+        assert self.project.slug in stacktrace_paths
+        assert sorted(stacktrace_paths[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/models/test_file.py",
+            "sentry/tasks.py",
+            "sentry/test_file.py",
+        ]