Add new task to find missing codemappings

src/sentry/tasks/find_missing_codemappings.py

@@ -0,0 +1,89 @@
+import logging
+from ast import Tuple
+from datetime import timedelta
+from typing import List
+
+from django.utils import timezone
+
+from sentry.db.models.fields.node import NodeData
+from sentry.models import Project
+from sentry.models.group import Group
+from sentry.models.organization import Organization, OrganizationStatus
+from sentry.tasks.base import instrumented_task
+from sentry.utils.safe import get_path
+
+PREFERRED_GROUP_OWNERS = 1
+PREFERRED_GROUP_OWNER_AGE = timedelta(days=7)
+
+logger = logging.getLogger("tasks.commit_context")
+
+
+@instrumented_task(
+    name="sentry.tasks.find_missing_codemappings",
+    queue="find_missing_codemappings",
+    max_retries=0,  # if we don't backfill it this time, we'll get it the next time
+)
+def find_missing_codemappings(**kwargs):
+    organizations = kwargs.get(
+        "organizations", Organization.objects.filter(status=OrganizationStatus.ACTIVE)
+    )
+
+    filename_maps = {}
+    for org in organizations:
+        projects = Project.objects.filter(organization=org, first_event__isnull=False)
+
+        projects = [
+            project
+            for project in projects
+            if Group.objects.filter(
+                project=project, last_seen__gte=timezone.now() - timedelta(days=7)
+            ).exists()
+        ]
+
+        project_file_map = {project.slug: get_all_filenames(project) for project in projects}
+        filename_maps[org.slug] = project_file_map
+    return filename_maps
+
+
+def get_all_filenames(project):
+    groups = Group.objects.filter(
+        project=project, last_seen__gte=timezone.now() - timedelta(days=14)
+    )
+
+    filenames = set()
+    for group in groups:
+        event = group.get_latest_event()
+        is_python_project, fn = get_filenames(project, event.data)
+        if not is_python_project:
+            return []
+        filenames.update(fn)
+
+    return list(filenames)
+
+
+# Get the filenames from the stacktrace for the latest event for an issue.
+def get_filenames(project: Project, data: NodeData) -> Tuple(bool, List[str]):
+    stacktraces = get_stacktrace(data)
+    filenames = set()
+    for st in stacktraces:
+        try:
+            fn = [frame["filename"] for frame in st["frames"]]
+            if fn[0].endswith(".py"):
+                filenames.update(fn)
+            else:
+                return False, []  # (is_python, filenames)
+        except Exception as e:
+            logger.log(logging.WARNING, f"Error getting filenames for project {project.slug}: {e}")
+    return True, filenames  # (is_python, filenames)
+
+
+def get_stacktrace(data: NodeData) -> List[str]:
+    exceptions = get_path(data, "exception", "values", filter=True)
+    if exceptions:
+        return [e["stacktrace"] for e in exceptions if get_path(e, "stacktrace", "frames")]
+
+    stacktrace = data.get("stacktrace")
+    if stacktrace and stacktrace.get("frames"):
+        return [stacktrace]
+
+    return None

tests/sentry/tasks/test_find_missing_codemappings.py

@@ -0,0 +1,295 @@
+from sentry.models.organization import OrganizationStatus
+from sentry.tasks.find_missing_codemappings import find_missing_codemappings
+from sentry.testutils import TestCase
+from sentry.testutils.helpers.datetime import before_now, iso_format
+
+
+class TestCommitContext(TestCase):
+    def setUp(self):
+        self.organization = self.create_organization(status=OrganizationStatus.ACTIVE)
+        self.project = self.create_project(organization=self.organization)
+
+    def test_finds_files_single_project(self):
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=1)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "handle_set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/tasks.py",
+                        },
+                        {
+                            "function": "set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/release.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=self.project.id,
+        )
+
+        with self.tasks():
+            mapping = find_missing_codemappings(organizations=[self.organization])
+        assert self.organization.slug in mapping
+        result = mapping[self.organization.slug]
+        assert self.project.slug in result
+        assert sorted(result[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+
+    def test_finds_files_multiple_projects(self):
+        project_1 = self.create_project(organization=self.organization)
+        project_2 = self.create_project(organization=self.organization)
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=1)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "handle_set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/tasks.py",
+                        },
+                        {
+                            "function": "set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/release.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=project_1.id,
+        )
+
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=2)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "test_fn",
+                            "abs_path": "/usr/src/sentry/src/sentry/test_file.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/test_file.py",
+                        },
+                        {
+                            "function": "test_fn_2",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/test_file.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/test_file.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=project_2.id,
+        )
+
+        with self.tasks():
+            mapping = find_missing_codemappings(organizations=[self.organization])
+        assert self.organization.slug in mapping
+        result = mapping[self.organization.slug]
+        assert project_1.slug in result
+        assert sorted(result[project_1.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+        assert project_2.slug in result
+        assert sorted(result[project_2.slug]) == [
+            "sentry/models/test_file.py",
+            "sentry/test_file.py",
+        ]
+
+    def test_finds_files_multiple_orgs(self):
+        new_org = self.create_organization()
+        new_project = self.create_project(organization=new_org)
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=1)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "handle_set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/tasks.py",
+                        },
+                        {
+                            "function": "set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/release.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=self.project.id,
+        )
+
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=2)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "test_fn",
+                            "abs_path": "/usr/src/sentry/src/sentry/test_file.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/test_file.py",
+                        },
+                        {
+                            "function": "test_fn_2",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/test_file.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/test_file.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=new_project.id,
+        )
+
+        with self.tasks():
+            mapping = find_missing_codemappings(organizations=[self.organization, new_org])
+        assert self.organization.slug in mapping
+        result_1 = mapping[self.organization.slug]
+        assert self.project.slug in result_1
+        assert sorted(result_1[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]
+        assert new_org.slug in mapping
+        result_2 = mapping[new_org.slug]
+        assert new_project.slug in result_2
+        assert sorted(result_2[new_project.slug]) == [
+            "sentry/models/test_file.py",
+            "sentry/test_file.py",
+        ]
+
+    def test_skips_stale_projects(self):
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=8)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "handle_set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/tasks.py",
+                        },
+                        {
+                            "function": "set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/release.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=self.project.id,
+        )
+
+        with self.tasks():
+            mapping = find_missing_codemappings()
+        assert self.organization.slug in mapping
+        result = mapping[self.organization.slug]
+        assert self.project.slug not in result
+
+    def test_handles_duplicates(self):
+        self.store_event(
+            data={
+                "message": "Kaboom!",
+                "platform": "python",
+                "timestamp": iso_format(before_now(days=1)),
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "function": "handle_set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 30,
+                            "filename": "sentry/tasks.py",
+                        },
+                        {
+                            "function": "set_commits",
+                            "abs_path": "/usr/src/sentry/src/sentry/models/release.py",
+                            "module": "sentry.models.release",
+                            "in_app": True,
+                            "lineno": 39,
+                            "filename": "sentry/models/release.py",
+                        },
+                        {
+                            "function": "handle_set_commits_new",
+                            "abs_path": "/usr/src/sentry/src/sentry/tasks.py",
+                            "module": "sentry.tasks",
+                            "in_app": False,
+                            "lineno": 40,
+                            "filename": "sentry/tasks.py",
+                        },
+                    ]
+                },
+                "fingerprint": ["put-me-in-the-control-group"],
+            },
+            project_id=self.project.id,
+        )
+
+        with self.tasks():
+            mapping = find_missing_codemappings(organizations=[self.organization])
+        assert self.organization.slug in mapping
+        result = mapping[self.organization.slug]
+        assert self.project.slug in result
+        assert sorted(result[self.project.slug]) == [
+            "sentry/models/release.py",
+            "sentry/tasks.py",
+        ]