diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py index a8ef6e2b50daf3..a29be8c95c4841 100644 --- a/src/sentry/seer/autofix/issue_summary.py +++ b/src/sentry/seer/autofix/issue_summary.py @@ -51,6 +51,13 @@ SeerAutomationSource.POST_PROCESS: "issue_summary_on_post_process_fixability", } +STOPPING_POINT_HIERARCHY = { + AutofixStoppingPoint.ROOT_CAUSE: 1, + AutofixStoppingPoint.SOLUTION: 2, + AutofixStoppingPoint.CODE_CHANGES: 3, + AutofixStoppingPoint.OPEN_PR: 4, +} + def _get_stopping_point_from_fixability(fixability_score: float) -> AutofixStoppingPoint | None: """ @@ -64,6 +71,58 @@ def _get_stopping_point_from_fixability(fixability_score: float) -> AutofixStopp return AutofixStoppingPoint.CODE_CHANGES +def _fetch_user_preference(project_id: int) -> str | None: + """ + Fetch the user's automated_run_stopping_point preference from Seer. + Returns None if preference is not set or if the API call fails. + """ + try: + path = "/v1/project-preference" + body = orjson.dumps({"project_id": project_id}) + + response = requests.post( + f"{settings.SEER_AUTOFIX_URL}{path}", + data=body, + headers={ + "content-type": "application/json;charset=utf-8", + **sign_with_seer_secret(body), + }, + timeout=5, + ) + response.raise_for_status() + + result = response.json() + preference = result.get("preference") + if preference: + return preference.get("automated_run_stopping_point") + return None + except Exception as e: + sentry_sdk.set_context("project", {"project_id": project_id}) + sentry_sdk.capture_exception(e) + return None + + +def _apply_user_preference_upper_bound( + fixability_suggestion: AutofixStoppingPoint | None, + user_preference: str | None, +) -> AutofixStoppingPoint | None: + """ + Apply user preference as an upper bound on the fixability-based stopping point. + Returns the more conservative (earlier) stopping point between the two. + """ + if fixability_suggestion is None or user_preference is None: + return fixability_suggestion + + user_stopping_point = AutofixStoppingPoint(user_preference) + + return ( + fixability_suggestion + if STOPPING_POINT_HIERARCHY[fixability_suggestion] + <= STOPPING_POINT_HIERARCHY[user_stopping_point] + else user_stopping_point + ) + + @instrumented_task( name="sentry.tasks.autofix.trigger_autofix_from_issue_summary", namespace=seer_tasks, @@ -277,8 +336,19 @@ def _run_automation( stopping_point = None if features.has("projects:triage-signals-v0", group.project): - stopping_point = _get_stopping_point_from_fixability(issue_summary.scores.fixability_score) - logger.info("Fixability-based stopping point: %s", stopping_point) + fixability_stopping_point = _get_stopping_point_from_fixability( + issue_summary.scores.fixability_score + ) + logger.info("Fixability-based stopping point: %s", fixability_stopping_point) + + # Fetch user preference and apply as upper bound + user_preference = _fetch_user_preference(group.project.id) + logger.info("User preference stopping point: %s", user_preference) + + stopping_point = _apply_user_preference_upper_bound( + fixability_stopping_point, user_preference + ) + logger.info("Final stopping point after upper bound: %s", stopping_point) _trigger_autofix_task.delay( group_id=group.id, diff --git a/tests/sentry/seer/autofix/test_issue_summary.py b/tests/sentry/seer/autofix/test_issue_summary.py index 60c9a38f7652a4..f2e48cefabe558 100644 --- a/tests/sentry/seer/autofix/test_issue_summary.py +++ b/tests/sentry/seer/autofix/test_issue_summary.py @@ -13,7 +13,9 @@ from sentry.locks import locks from sentry.seer.autofix.constants import SeerAutomationSource from sentry.seer.autofix.issue_summary import ( + _apply_user_preference_upper_bound, _call_seer, + _fetch_user_preference, _get_event, _get_stopping_point_from_fixability, _run_automation, @@ -789,3 +791,215 @@ def test_without_feature_flag(self, mock_gen, mock_budget, mock_state, mock_rate mock_trigger.assert_called_once() assert mock_trigger.call_args[1]["stopping_point"] is None + + +class TestFetchUserPreference: + @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={}) + @patch("sentry.seer.autofix.issue_summary.requests.post") + def test_fetch_user_preference_success(self, mock_post, mock_sign): + mock_response = Mock() + mock_response.json.return_value = { + "preference": {"automated_run_stopping_point": "solution"} + } + mock_response.raise_for_status = Mock() + mock_post.return_value = mock_response + + result = _fetch_user_preference(project_id=123) + + assert result == "solution" + mock_post.assert_called_once() + mock_response.raise_for_status.assert_called_once() + + @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={}) + @patch("sentry.seer.autofix.issue_summary.requests.post") + def test_fetch_user_preference_no_preference(self, mock_post, mock_sign): + mock_response = Mock() + mock_response.json.return_value = {"preference": None} + mock_response.raise_for_status = Mock() + mock_post.return_value = mock_response + + result = _fetch_user_preference(project_id=123) + + assert result is None + + @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={}) + @patch("sentry.seer.autofix.issue_summary.requests.post") + def test_fetch_user_preference_empty_preference(self, mock_post, mock_sign): + mock_response = Mock() + mock_response.json.return_value = {"preference": {"automated_run_stopping_point": None}} + mock_response.raise_for_status = Mock() + mock_post.return_value = mock_response + + result = _fetch_user_preference(project_id=123) + + assert result is None + + @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={}) + @patch("sentry.seer.autofix.issue_summary.requests.post") + def test_fetch_user_preference_api_error(self, mock_post, mock_sign): + mock_post.side_effect = Exception("API error") + + result = _fetch_user_preference(project_id=123) + + assert result is None + + +class TestApplyUserPreferenceUpperBound: + @pytest.mark.parametrize( + "fixability,user_pref,expected", + [ + # Fixability is None - always return None + (None, "open_pr", None), + (None, "solution", None), + (None, None, None), + # User preference is None - return fixability suggestion + (AutofixStoppingPoint.OPEN_PR, None, AutofixStoppingPoint.OPEN_PR), + (AutofixStoppingPoint.CODE_CHANGES, None, AutofixStoppingPoint.CODE_CHANGES), + (AutofixStoppingPoint.SOLUTION, None, AutofixStoppingPoint.SOLUTION), + (AutofixStoppingPoint.ROOT_CAUSE, None, AutofixStoppingPoint.ROOT_CAUSE), + # User preference limits automation (user is more conservative) + ( + AutofixStoppingPoint.OPEN_PR, + "code_changes", + AutofixStoppingPoint.CODE_CHANGES, + ), + (AutofixStoppingPoint.OPEN_PR, "solution", AutofixStoppingPoint.SOLUTION), + (AutofixStoppingPoint.OPEN_PR, "root_cause", AutofixStoppingPoint.ROOT_CAUSE), + (AutofixStoppingPoint.CODE_CHANGES, "solution", AutofixStoppingPoint.SOLUTION), + ( + AutofixStoppingPoint.CODE_CHANGES, + "root_cause", + AutofixStoppingPoint.ROOT_CAUSE, + ), + (AutofixStoppingPoint.SOLUTION, "root_cause", AutofixStoppingPoint.ROOT_CAUSE), + # Fixability is more conservative (fixability limits automation) + (AutofixStoppingPoint.SOLUTION, "open_pr", AutofixStoppingPoint.SOLUTION), + ( + AutofixStoppingPoint.SOLUTION, + "code_changes", + AutofixStoppingPoint.SOLUTION, + ), + (AutofixStoppingPoint.ROOT_CAUSE, "open_pr", AutofixStoppingPoint.ROOT_CAUSE), + ( + AutofixStoppingPoint.ROOT_CAUSE, + "code_changes", + AutofixStoppingPoint.ROOT_CAUSE, + ), + (AutofixStoppingPoint.ROOT_CAUSE, "solution", AutofixStoppingPoint.ROOT_CAUSE), + # Same level - return fixability + (AutofixStoppingPoint.OPEN_PR, "open_pr", AutofixStoppingPoint.OPEN_PR), + ( + AutofixStoppingPoint.CODE_CHANGES, + "code_changes", + AutofixStoppingPoint.CODE_CHANGES, + ), + (AutofixStoppingPoint.SOLUTION, "solution", AutofixStoppingPoint.SOLUTION), + ( + AutofixStoppingPoint.ROOT_CAUSE, + "root_cause", + AutofixStoppingPoint.ROOT_CAUSE, + ), + ], + ) + def test_upper_bound_combinations(self, fixability, user_pref, expected): + result = _apply_user_preference_upper_bound(fixability, user_pref) + assert result == expected + + +@with_feature({"organizations:gen-ai-features": True, "projects:triage-signals-v0": True}) +class TestRunAutomationWithUpperBound(APITestCase, SnubaTestCase): + def setUp(self) -> None: + super().setUp() + self.group = self.create_group() + event_data = load_data("python") + self.event = self.store_event(data=event_data, project_id=self.project.id) + + @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay") + @patch("sentry.seer.autofix.issue_summary._fetch_user_preference") + @patch( + "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited", + return_value=False, + ) + @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None) + @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True) + @patch("sentry.seer.autofix.issue_summary._generate_fixability_score") + def test_user_preference_limits_high_fixability( + self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger + ): + """High fixability (CODE_CHANGES) limited by user preference (SOLUTION)""" + self.project.update_option("sentry:autofix_automation_tuning", "always") + mock_gen.return_value = SummarizeIssueResponse( + group_id=str(self.group.id), + headline="h", + whats_wrong="w", + trace="t", + possible_cause="c", + scores=SummarizeIssueScores(fixability_score=0.80), # High = CODE_CHANGES + ) + mock_fetch.return_value = "solution" + + _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT) + + mock_trigger.assert_called_once() + # Should be limited to SOLUTION by user preference + assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.SOLUTION + + @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay") + @patch("sentry.seer.autofix.issue_summary._fetch_user_preference") + @patch( + "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited", + return_value=False, + ) + @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None) + @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True) + @patch("sentry.seer.autofix.issue_summary._generate_fixability_score") + def test_fixability_limits_permissive_user_preference( + self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger + ): + """Medium fixability (SOLUTION) used despite user allowing OPEN_PR""" + self.project.update_option("sentry:autofix_automation_tuning", "always") + mock_gen.return_value = SummarizeIssueResponse( + group_id=str(self.group.id), + headline="h", + whats_wrong="w", + trace="t", + possible_cause="c", + scores=SummarizeIssueScores(fixability_score=0.50), # Medium = SOLUTION + ) + mock_fetch.return_value = "open_pr" + + _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT) + + mock_trigger.assert_called_once() + # Should use SOLUTION from fixability, not OPEN_PR from user + assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.SOLUTION + + @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay") + @patch("sentry.seer.autofix.issue_summary._fetch_user_preference") + @patch( + "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited", + return_value=False, + ) + @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None) + @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True) + @patch("sentry.seer.autofix.issue_summary._generate_fixability_score") + def test_no_user_preference_uses_fixability_only( + self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger + ): + """When user has no preference, use fixability score alone""" + self.project.update_option("sentry:autofix_automation_tuning", "always") + mock_gen.return_value = SummarizeIssueResponse( + group_id=str(self.group.id), + headline="h", + whats_wrong="w", + trace="t", + possible_cause="c", + scores=SummarizeIssueScores(fixability_score=0.80), # High = CODE_CHANGES + ) + mock_fetch.return_value = None + + _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT) + + mock_trigger.assert_called_once() + # Should use CODE_CHANGES from fixability + assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.CODE_CHANGES