From 4db9bb21548dee469f0571240174ad6dba83c5d5 Mon Sep 17 00:00:00 2001 From: Josh Ferge Date: Mon, 17 Nov 2025 18:04:55 -0500 Subject: [PATCH 1/2] feat(seer): Add separate scanner acknowledgement function with rollout rate - Add `get_seer_org_acknowledgement_for_scanner` function that uses deterministic rollout via `in_rollout_group` instead of random.random() - Add `seer.scanner_no_consent.rollout_rate` option for controlling scanner rollout - Update `is_issue_eligible_for_seer_automation` to use new function - Add comprehensive unit tests for `get_seer_org_acknowledgement_for_scanner` - Add unit tests for `is_issue_eligible_for_seer_automation` - Fix duplicate return statements in `has_seer_access_with_detail` The new function allows independent control of scanner behavior with a deterministic rollout based on organization ID, separate from the general Seer acknowledgement logic. --- src/sentry/options/defaults.py | 7 + src/sentry/seer/autofix/utils.py | 4 +- src/sentry/seer/seer_setup.py | 18 ++ .../sentry/seer/autofix/test_autofix_utils.py | 123 ++++++++++ tests/sentry/seer/test_seer_setup.py | 210 ++++++++++++++++++ 5 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 tests/sentry/seer/test_seer_setup.py diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index c32789b431493c..48f9d05d212116 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -3655,3 +3655,10 @@ default=0.0, flags=FLAG_AUTOMATOR_MODIFIABLE, ) + +register( + "seer.scanner_no_consent.rollout_rate", + type=Float, + default=0.0, + flags=FLAG_MODIFIABLE_RATE | FLAG_AUTOMATOR_MODIFIABLE, +) diff --git a/src/sentry/seer/autofix/utils.py b/src/sentry/seer/autofix/utils.py index 31e0da5e47c6b5..f11e2e2d00249b 100644 --- a/src/sentry/seer/autofix/utils.py +++ b/src/sentry/seer/autofix/utils.py @@ -320,9 +320,9 @@ def is_issue_eligible_for_seer_automation(group: Group) -> bool: ): return False - from sentry.seer.seer_setup import get_seer_org_acknowledgement + from sentry.seer.seer_setup import get_seer_org_acknowledgement_for_scanner - seer_enabled = get_seer_org_acknowledgement(group.organization) + seer_enabled = get_seer_org_acknowledgement_for_scanner(group.organization) if not seer_enabled: return False diff --git a/src/sentry/seer/seer_setup.py b/src/sentry/seer/seer_setup.py index f6a2db46cf845a..32471d933cc50d 100644 --- a/src/sentry/seer/seer_setup.py +++ b/src/sentry/seer/seer_setup.py @@ -3,6 +3,7 @@ from sentry import features from sentry.models.organization import Organization from sentry.models.promptsactivity import PromptsActivity +from sentry.options.rollout import in_rollout_group from sentry.users.models.user import User from sentry.users.services.user.model import RpcUser @@ -36,6 +37,23 @@ def get_seer_org_acknowledgement(organization: Organization) -> bool: ).exists() +def get_seer_org_acknowledgement_for_scanner(organization: Organization) -> bool: + + if PromptsActivity.objects.filter( + feature=feature_name, + organization_id=organization.id, + project_id=0, + ).exists(): + return True + + if features.has("organizations:gen-ai-consent-flow-removal", organization) and in_rollout_group( + "seer.scanner_no_consent.rollout_rate", organization.id + ): + return True + + return False + + def has_seer_access( organization: Organization, actor: User | AnonymousUser | RpcUser | None = None ) -> bool: diff --git a/tests/sentry/seer/autofix/test_autofix_utils.py b/tests/sentry/seer/autofix/test_autofix_utils.py index 598302b3d38662..fb87fd08c0f2c9 100644 --- a/tests/sentry/seer/autofix/test_autofix_utils.py +++ b/tests/sentry/seer/autofix/test_autofix_utils.py @@ -3,6 +3,7 @@ import orjson import pytest +from sentry.constants import DataCategory from sentry.seer.autofix.constants import AutofixStatus from sentry.seer.autofix.utils import ( AutofixState, @@ -10,6 +11,7 @@ CodingAgentStatus, get_autofix_prompt, get_coding_agent_prompt, + is_issue_eligible_for_seer_automation, ) from sentry.seer.models import SeerApiError from sentry.testutils.cases import TestCase @@ -189,3 +191,124 @@ def test_autofix_state_validate_parses_nested_structures(self): # Top-level coding_agents map is parsed with enum status assert state.coding_agents["agent-1"].status == CodingAgentStatus.COMPLETED + + +class TestIsIssueEligibleForSeerAutomation(TestCase): + """Test the is_issue_eligible_for_seer_automation function.""" + + def setUp(self): + super().setUp() + self.organization = self.create_organization(name="test-org") + self.project = self.create_project(organization=self.organization) + self.group = self.create_group(project=self.project) + + def test_returns_false_for_unsupported_issue_categories(self): + """Test returns False for unsupported issue categories like REPLAY and FEEDBACK.""" + from sentry.issues.grouptype import FeedbackGroup, ReplayRageClickType + + # Create groups with unsupported categories + replay_group = self.create_group(project=self.project, type=ReplayRageClickType.type_id) + feedback_group = self.create_group(project=self.project, type=FeedbackGroup.type_id) + + assert is_issue_eligible_for_seer_automation(replay_group) is False + assert is_issue_eligible_for_seer_automation(feedback_group) is False + + def test_returns_true_for_supported_issue_categories(self): + """Test returns True for supported issue categories when all conditions are met.""" + with self.feature("organizations:gen-ai-features"): + with patch( + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner" + ) as mock_ack: + with patch("sentry.quotas.backend.has_available_reserved_budget") as mock_budget: + mock_ack.return_value = True + mock_budget.return_value = True + self.project.update_option("sentry:seer_scanner_automation", True) + + # Test supported categories - using default error group + result = is_issue_eligible_for_seer_automation(self.group) + + assert result is True + + def test_returns_false_when_gen_ai_features_not_enabled(self): + """Test returns False when organizations:gen-ai-features feature flag is not enabled.""" + result = is_issue_eligible_for_seer_automation(self.group) + assert result is False + + def test_returns_false_when_ai_features_hidden(self): + """Test returns False when sentry:hide_ai_features option is enabled.""" + with self.feature("organizations:gen-ai-features"): + self.organization.update_option("sentry:hide_ai_features", True) + result = is_issue_eligible_for_seer_automation(self.group) + assert result is False + + def test_returns_false_when_scanner_automation_disabled_and_not_always_trigger(self): + """Test returns False when scanner automation is disabled and issue type doesn't always trigger.""" + with self.feature("organizations:gen-ai-features"): + self.project.update_option("sentry:seer_scanner_automation", False) + result = is_issue_eligible_for_seer_automation(self.group) + assert result is False + + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner") + def test_returns_false_when_org_not_acknowledged(self, mock_get_acknowledgement): + """Test returns False when organization has not acknowledged Seer for scanner.""" + with self.feature("organizations:gen-ai-features"): + self.project.update_option("sentry:seer_scanner_automation", True) + mock_get_acknowledgement.return_value = False + + result = is_issue_eligible_for_seer_automation(self.group) + + assert result is False + mock_get_acknowledgement.assert_called_once_with(self.organization) + + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner") + @patch("sentry.quotas.backend.has_available_reserved_budget") + def test_returns_false_when_no_budget_available( + self, mock_has_budget, mock_get_acknowledgement + ): + """Test returns False when organization has no available budget for scanner.""" + with self.feature("organizations:gen-ai-features"): + self.project.update_option("sentry:seer_scanner_automation", True) + mock_get_acknowledgement.return_value = True + mock_has_budget.return_value = False + + result = is_issue_eligible_for_seer_automation(self.group) + + assert result is False + mock_has_budget.assert_called_once_with( + org_id=self.organization.id, data_category=DataCategory.SEER_SCANNER + ) + + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner") + @patch("sentry.quotas.backend.has_available_reserved_budget") + def test_returns_true_when_all_conditions_met(self, mock_has_budget, mock_get_acknowledgement): + """Test returns True when all eligibility conditions are met.""" + with self.feature("organizations:gen-ai-features"): + self.project.update_option("sentry:seer_scanner_automation", True) + mock_get_acknowledgement.return_value = True + mock_has_budget.return_value = True + + result = is_issue_eligible_for_seer_automation(self.group) + + assert result is True + mock_get_acknowledgement.assert_called_once_with(self.organization) + mock_has_budget.assert_called_once_with( + org_id=self.organization.id, data_category=DataCategory.SEER_SCANNER + ) + + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner") + @patch("sentry.quotas.backend.has_available_reserved_budget") + def test_returns_true_when_issue_type_always_triggers( + self, mock_has_budget, mock_get_acknowledgement + ): + """Test returns True when issue type has always_trigger_seer_automation even if scanner automation is disabled.""" + with self.feature("organizations:gen-ai-features"): + # Disable scanner automation + self.project.update_option("sentry:seer_scanner_automation", False) + mock_get_acknowledgement.return_value = True + mock_has_budget.return_value = True + + # Mock the group's issue_type to always trigger + with patch.object(self.group.issue_type, "always_trigger_seer_automation", True): + result = is_issue_eligible_for_seer_automation(self.group) + + assert result is True diff --git a/tests/sentry/seer/test_seer_setup.py b/tests/sentry/seer/test_seer_setup.py new file mode 100644 index 00000000000000..c9c756ad8207d5 --- /dev/null +++ b/tests/sentry/seer/test_seer_setup.py @@ -0,0 +1,210 @@ +from unittest.mock import patch + +import orjson + +from sentry.models.promptsactivity import PromptsActivity +from sentry.seer.seer_setup import ( + get_seer_org_acknowledgement, + get_seer_org_acknowledgement_for_scanner, + get_seer_user_acknowledgement, +) +from sentry.testutils.cases import TestCase + + +class TestGetSeerOrgAcknowledgementForScanner(TestCase): + def setUp(self): + super().setUp() + self.organization = self.create_organization(name="test-org") + self.user = self.create_user() + self.feature_name = "seer_autofix_setup_acknowledged" + + def test_returns_true_when_org_has_acknowledged(self): + """Test returns True when organization has acknowledged via PromptsActivity.""" + PromptsActivity.objects.create( + user_id=self.user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + assert result is True + + def test_returns_false_when_no_acknowledgement_and_feature_not_enabled(self): + """Test returns False when no acknowledgement exists and feature flag is disabled.""" + result = get_seer_org_acknowledgement_for_scanner(self.organization) + assert result is False + + @patch("sentry.seer.seer_setup.in_rollout_group") + def test_returns_true_when_feature_enabled_and_passes_rollout(self, mock_in_rollout_group): + """Test returns True when gen-ai-consent-flow-removal is enabled and passes rollout rate.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + mock_in_rollout_group.return_value = True + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + + assert result is True + mock_in_rollout_group.assert_called_once_with( + "seer.scanner_no_consent.rollout_rate", self.organization.id + ) + + @patch("sentry.seer.seer_setup.in_rollout_group") + def test_returns_false_when_feature_enabled_but_fails_rollout(self, mock_in_rollout_group): + """Test returns False when gen-ai-consent-flow-removal is enabled but fails rollout rate.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + mock_in_rollout_group.return_value = False + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + + assert result is False + mock_in_rollout_group.assert_called_once_with( + "seer.scanner_no_consent.rollout_rate", self.organization.id + ) + + @patch("sentry.seer.seer_setup.in_rollout_group") + def test_returns_true_when_feature_enabled_and_100_percent_rollout(self, mock_in_rollout_group): + """Test returns True when gen-ai-consent-flow-removal is enabled with 100% rollout.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + mock_in_rollout_group.return_value = True + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + + assert result is True + mock_in_rollout_group.assert_called_once_with( + "seer.scanner_no_consent.rollout_rate", self.organization.id + ) + + @patch("sentry.seer.seer_setup.in_rollout_group") + def test_returns_false_when_feature_enabled_and_0_percent_rollout(self, mock_in_rollout_group): + """Test returns False when gen-ai-consent-flow-removal is enabled with 0% rollout.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + mock_in_rollout_group.return_value = False + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + + assert result is False + mock_in_rollout_group.assert_called_once_with( + "seer.scanner_no_consent.rollout_rate", self.organization.id + ) + + @patch("sentry.seer.seer_setup.in_rollout_group") + def test_prioritizes_acknowledgement_over_feature_flag(self, mock_in_rollout_group): + """Test that explicit acknowledgement takes priority over feature flag rollout.""" + PromptsActivity.objects.create( + user_id=self.user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + with self.feature("organizations:gen-ai-consent-flow-removal"): + mock_in_rollout_group.return_value = False + + result = get_seer_org_acknowledgement_for_scanner(self.organization) + + assert result is True + # Should return True before checking rollout + mock_in_rollout_group.assert_not_called() + + def test_different_organizations_isolated(self): + """Test that acknowledgements are isolated per organization.""" + org2 = self.create_organization(name="test-org-2") + + PromptsActivity.objects.create( + user_id=self.user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + result1 = get_seer_org_acknowledgement_for_scanner(self.organization) + result2 = get_seer_org_acknowledgement_for_scanner(org2) + + assert result1 is True + assert result2 is False + + +class TestGetSeerOrgAcknowledgement(TestCase): + """Test the standard get_seer_org_acknowledgement function for comparison.""" + + def setUp(self): + super().setUp() + self.organization = self.create_organization(name="test-org") + self.user = self.create_user() + self.feature_name = "seer_autofix_setup_acknowledged" + + def test_returns_true_when_gen_ai_consent_removal_enabled(self): + """Test returns True when gen-ai-consent-flow-removal feature is enabled.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + result = get_seer_org_acknowledgement(self.organization) + assert result is True + + def test_returns_true_when_org_has_acknowledged(self): + """Test returns True when organization has acknowledged via PromptsActivity.""" + PromptsActivity.objects.create( + user_id=self.user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + result = get_seer_org_acknowledgement(self.organization) + assert result is True + + def test_returns_false_when_no_acknowledgement_and_feature_not_enabled(self): + """Test returns False when no acknowledgement exists and feature flag is disabled.""" + result = get_seer_org_acknowledgement(self.organization) + assert result is False + + +class TestGetSeerUserAcknowledgement(TestCase): + """Test the get_seer_user_acknowledgement function.""" + + def setUp(self): + super().setUp() + self.organization = self.create_organization(name="test-org") + self.user = self.create_user() + self.feature_name = "seer_autofix_setup_acknowledged" + + def test_returns_true_when_gen_ai_consent_removal_enabled(self): + """Test returns True when gen-ai-consent-flow-removal feature is enabled.""" + with self.feature("organizations:gen-ai-consent-flow-removal"): + result = get_seer_user_acknowledgement(self.user.id, self.organization) + assert result is True + + def test_returns_true_when_user_has_acknowledged(self): + """Test returns True when user has acknowledged via PromptsActivity.""" + PromptsActivity.objects.create( + user_id=self.user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + result = get_seer_user_acknowledgement(self.user.id, self.organization) + assert result is True + + def test_returns_false_when_no_acknowledgement_and_feature_not_enabled(self): + """Test returns False when user has not acknowledged and feature flag is disabled.""" + result = get_seer_user_acknowledgement(self.user.id, self.organization) + assert result is False + + def test_returns_false_when_different_user_acknowledged(self): + """Test returns False when a different user has acknowledged.""" + other_user = self.create_user() + + PromptsActivity.objects.create( + user_id=other_user.id, + feature=self.feature_name, + organization_id=self.organization.id, + project_id=0, + data=orjson.dumps({"dismissed_ts": 123456789}).decode("utf-8"), + ) + + result = get_seer_user_acknowledgement(self.user.id, self.organization) + assert result is False From 9d60fa00bdda55ec065f95315744cad30875fe81 Mon Sep 17 00:00:00 2001 From: Josh Ferge Date: Mon, 17 Nov 2025 18:24:41 -0500 Subject: [PATCH 2/2] refactor(seer): Simplify return and fix test patches - Simplify get_seer_org_acknowledgement_for_scanner to use a single return statement - Update all test patches from get_seer_org_acknowledgement to get_seer_org_acknowledgement_for_scanner - Fix failing tests in test_post_process.py --- src/sentry/seer/seer_setup.py | 16 +++++----------- tests/sentry/tasks/test_post_process.py | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/sentry/seer/seer_setup.py b/src/sentry/seer/seer_setup.py index 32471d933cc50d..d15597e5f0a6d8 100644 --- a/src/sentry/seer/seer_setup.py +++ b/src/sentry/seer/seer_setup.py @@ -38,20 +38,14 @@ def get_seer_org_acknowledgement(organization: Organization) -> bool: def get_seer_org_acknowledgement_for_scanner(organization: Organization) -> bool: - - if PromptsActivity.objects.filter( + return PromptsActivity.objects.filter( feature=feature_name, organization_id=organization.id, project_id=0, - ).exists(): - return True - - if features.has("organizations:gen-ai-consent-flow-removal", organization) and in_rollout_group( - "seer.scanner_no_consent.rollout_rate", organization.id - ): - return True - - return False + ).exists() or ( + features.has("organizations:gen-ai-consent-flow-removal", organization) + and in_rollout_group("seer.scanner_no_consent.rollout_rate", organization.id) + ) def has_seer_access( diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index f8805c400715fa..4fdaddaca4abf3 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -2672,7 +2672,7 @@ def test_skip_process_similarity_global(self, mock_safe_execute: MagicMock) -> N class KickOffSeerAutomationTestMixin(BasePostProgressGroupMixin): @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2696,7 +2696,7 @@ def test_kick_off_seer_automation_with_features( mock_start_seer_automation.assert_called_once_with(event.group.id) @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2718,7 +2718,7 @@ def test_kick_off_seer_automation_without_org_feature( mock_start_seer_automation.assert_not_called() @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=False, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2742,7 +2742,7 @@ def test_kick_off_seer_automation_without_seer_enabled( mock_start_seer_automation.assert_not_called() @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2767,7 +2767,7 @@ def test_kick_off_seer_automation_without_scanner_on( mock_start_seer_automation.assert_not_called() @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2796,7 +2796,7 @@ def test_kick_off_seer_automation_skips_existing_fixability_score( mock_start_seer_automation.assert_not_called() @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2824,7 +2824,7 @@ def test_kick_off_seer_automation_runs_with_missing_fixability_score( mock_start_seer_automation.assert_called_once_with(group.id) @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -2860,7 +2860,7 @@ def test_kick_off_seer_automation_skips_with_existing_fixability_score( @patch("sentry.seer.autofix.utils.is_seer_scanner_rate_limited") @patch("sentry.quotas.backend.has_available_reserved_budget") - @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement") + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner") @patch("sentry.tasks.autofix.start_seer_automation.delay") @with_feature("organizations:gen-ai-features") def test_rate_limit_only_checked_after_all_other_checks_pass( @@ -2954,7 +2954,7 @@ def test_rate_limit_only_checked_after_all_other_checks_pass( mock_start_seer_automation.assert_not_called() @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -3005,7 +3005,7 @@ def test_kick_off_seer_automation_skips_when_lock_held( mock_start_seer_automation.assert_called_once_with(event2.group.id) @patch( - "sentry.seer.seer_setup.get_seer_org_acknowledgement", + "sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True, ) @patch("sentry.tasks.autofix.start_seer_automation.delay") @@ -3036,7 +3036,7 @@ class SeerAutomationHelperFunctionsTestMixin(BasePostProgressGroupMixin): """Unit tests for is_issue_eligible_for_seer_automation.""" @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True) - @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement", return_value=True) + @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement_for_scanner", return_value=True) @patch("sentry.features.has", return_value=True) def test_is_issue_eligible_for_seer_automation( self, mock_features_has, mock_get_seer_org_acknowledgement, mock_has_budget