From 5dc20aeca05ec7632228fe6d632aebb3a4264965 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:00:10 -0400 Subject: [PATCH 1/8] feat(seer): Allow night shift strategy and max candidate overrides Add a triage strategy registry plus a new `seer.night_shift.default_strategy` option so we can swap strategies (e.g. future `agentic_triage_v3`) without a deploy. The cron resolves strategy and max_candidates from options; the admin trigger endpoint can override both per-call for easier testing. Also fix `seer.night_shift.issues_per_org`, which was dormant and drifting (option default was 5, hardcoded cap was 10). The option is now actually read and defaults to 10. Admin endpoint validates unknown strategies (400) and clamps max_candidates to [1, 50]. --- src/sentry/options/defaults.py | 8 ++- .../endpoints/admin_night_shift_trigger.py | 34 +++++++++++- .../tasks/seer/night_shift/agentic_triage.py | 3 +- src/sentry/tasks/seer/night_shift/cron.py | 20 +++++-- .../tasks/seer/night_shift/simple_triage.py | 4 +- .../tasks/seer/night_shift/strategies.py | 40 ++++++++++++++ .../test_admin_night_shift_trigger.py | 53 ++++++++++++++++++- tests/sentry/tasks/seer/test_night_shift.py | 17 +++--- .../tasks/seer/test_night_shift_strategies.py | 37 +++++++++++++ 9 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 src/sentry/tasks/seer/night_shift/strategies.py create mode 100644 tests/sentry/tasks/seer/test_night_shift_strategies.py diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 1612f21b8e2f..4397c0dd0f79 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -1377,7 +1377,13 @@ ) register( "seer.night_shift.issues_per_org", - default=5, + default=10, + flags=FLAG_AUTOMATOR_MODIFIABLE, +) +register( + "seer.night_shift.default_strategy", + type=String, + default="agentic_triage", flags=FLAG_AUTOMATOR_MODIFIABLE, ) diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index 24153fbf99ea..9c7e920e4b89 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -6,6 +6,9 @@ from sentry.api.base import Endpoint, internal_cell_silo_endpoint from sentry.api.permissions import StaffPermission from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org +from sentry.tasks.seer.night_shift.strategies import TRIAGE_STRATEGIES + +MAX_CANDIDATES_CEILING = 50 @internal_cell_silo_endpoint @@ -28,12 +31,41 @@ def post(self, request: Request) -> Response: dry_run = bool(request.data.get("dry_run", False)) - run_night_shift_for_org.apply_async(args=[organization_id], kwargs={"dry_run": dry_run}) + strategy = request.data.get("strategy") or None + if strategy is not None and strategy not in TRIAGE_STRATEGIES: + return Response( + { + "detail": f"unknown strategy {strategy!r}; valid options: {sorted(TRIAGE_STRATEGIES)}" + }, + status=400, + ) + + max_candidates_raw = request.data.get("max_candidates") + max_candidates: int | None + if max_candidates_raw in (None, ""): + max_candidates = None + else: + try: + max_candidates = int(max_candidates_raw) + except (ValueError, TypeError): + return Response({"detail": "max_candidates must be a valid integer"}, status=400) + max_candidates = max(1, min(max_candidates, MAX_CANDIDATES_CEILING)) + + run_night_shift_for_org.apply_async( + args=[organization_id], + kwargs={ + "dry_run": dry_run, + "strategy": strategy, + "max_candidates": max_candidates, + }, + ) return Response( { "success": True, "organization_id": organization_id, "dry_run": dry_run, + "strategy": strategy, + "max_candidates": max_candidates, } ) diff --git a/src/sentry/tasks/seer/night_shift/agentic_triage.py b/src/sentry/tasks/seer/night_shift/agentic_triage.py index 689fc28358af..77bc5ae919a2 100644 --- a/src/sentry/tasks/seer/night_shift/agentic_triage.py +++ b/src/sentry/tasks/seer/night_shift/agentic_triage.py @@ -35,6 +35,7 @@ class _TriageResponse(pydantic.BaseModel): def agentic_triage_strategy( projects: Sequence[Project], organization: Organization, + max_candidates: int, ) -> tuple[list[TriageResult], int | None]: """ Select candidates via fixability scoring, then use the Seer Explorer agent @@ -43,7 +44,7 @@ def agentic_triage_strategy( Returns a tuple of (triage_results, agent_run_id). """ # TODO: try a new way to get scored issues - scored = fixability_score_strategy(projects) + scored = fixability_score_strategy(projects, max_candidates) if not scored: return [], None diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index 6afb5afe3810..012330e0b5ac 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -25,8 +25,8 @@ from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue from sentry.seer.models.seer_api_models import SeerProjectPreference from sentry.tasks.base import instrumented_task -from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy from sentry.tasks.seer.night_shift.models import TriageAction +from sentry.tasks.seer.night_shift.strategies import resolve_triage_strategy from sentry.taskworker.namespaces import seer_tasks from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -87,7 +87,12 @@ def schedule_night_shift() -> None: namespace=seer_tasks, processing_deadline_duration=5 * 60, ) -def run_night_shift_for_org(organization_id: int, dry_run: bool = False) -> None: +def run_night_shift_for_org( + organization_id: int, + dry_run: bool = False, + strategy: str | None = None, + max_candidates: int | None = None, +) -> None: try: organization = Organization.objects.get( id=organization_id, status=OrganizationStatus.ACTIVE @@ -126,7 +131,12 @@ def run_night_shift_for_org(organization_id: int, dry_run: bool = False) -> None sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible_projects)) - triage_strategy = "agentic_triage" + triage_strategy, strategy_fn = resolve_triage_strategy(strategy) + resolved_max_candidates = ( + max_candidates + if max_candidates is not None + else options.get("seer.night_shift.issues_per_org") + ) run = SeerNightShiftRun.objects.create( organization=organization, triage_strategy=triage_strategy, @@ -134,7 +144,9 @@ def run_night_shift_for_org(organization_id: int, dry_run: bool = False) -> None agent_run_id = None try: - candidates, agent_run_id = agentic_triage_strategy(eligible_projects, organization) + candidates, agent_run_id = strategy_fn( + eligible_projects, organization, resolved_max_candidates + ) if candidates: SeerNightShiftRunIssue.objects.bulk_create( diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index 5c3bb109e786..eaad266cf4ca 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -17,7 +17,6 @@ logger = logging.getLogger("sentry.tasks.seer.night_shift") -NIGHT_SHIFT_MAX_CANDIDATES = 10 NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100 # Weights for candidate scoring. Set to 0 to disable a signal. @@ -49,6 +48,7 @@ def __lt__(self, other: ScoredCandidate) -> bool: def fixability_score_strategy( projects: Sequence[Project], + max_candidates: int, ) -> list[ScoredCandidate]: """ Fetch top recommended unresolved issues that haven't been triaged by Seer yet, @@ -88,7 +88,7 @@ def fixability_score_strategy( ) candidates.sort(reverse=True) - selected = candidates[:NIGHT_SHIFT_MAX_CANDIDATES] + selected = candidates[:max_candidates] for c in selected: sentry_sdk.metrics.distribution("night_shift.fixability_score", c.fixability) diff --git a/src/sentry/tasks/seer/night_shift/strategies.py b/src/sentry/tasks/seer/night_shift/strategies.py new file mode 100644 index 000000000000..1a3adc196207 --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/strategies.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import logging +from collections.abc import Callable, Sequence + +from sentry import options +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy +from sentry.tasks.seer.night_shift.models import TriageResult + +logger = logging.getLogger("sentry.tasks.seer.night_shift") + +TriageStrategyFn = Callable[ + [Sequence[Project], Organization, int], + tuple[list[TriageResult], int | None], +] + +TRIAGE_STRATEGIES: dict[str, TriageStrategyFn] = { + "agentic_triage": agentic_triage_strategy, +} + +DEFAULT_TRIAGE_STRATEGY = "agentic_triage" + + +def resolve_triage_strategy(override: str | None) -> tuple[str, TriageStrategyFn]: + """ + Resolve a strategy name using: explicit override -> options -> constant default. + If the resolved name isn't in the registry, log and fall back to the default. + """ + name = override or options.get("seer.night_shift.default_strategy") or DEFAULT_TRIAGE_STRATEGY + fn = TRIAGE_STRATEGIES.get(name) + if fn is None: + logger.warning( + "night_shift.unknown_strategy", + extra={"requested": name, "fallback": DEFAULT_TRIAGE_STRATEGY}, + ) + name = DEFAULT_TRIAGE_STRATEGY + fn = TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] + return name, fn diff --git a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py index 70eb8fd993f8..e588d0badd54 100644 --- a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py +++ b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py @@ -28,10 +28,61 @@ def test_trigger_night_shift(self) -> None: assert response.data["success"] is True assert response.data["organization_id"] == self.organization.id + assert response.data["strategy"] is None + assert response.data["max_candidates"] is None mock_task.apply_async.assert_called_once_with( - args=[self.organization.id], kwargs={"dry_run": False} + args=[self.organization.id], + kwargs={"dry_run": False, "strategy": None, "max_candidates": None}, ) + def test_trigger_with_overrides(self) -> None: + with patch( + "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org" + ) as mock_task: + response = self.get_success_response( + organization_id=self.organization.id, + strategy="agentic_triage", + max_candidates=3, + dry_run=True, + status_code=200, + ) + + assert response.data["strategy"] == "agentic_triage" + assert response.data["max_candidates"] == 3 + mock_task.apply_async.assert_called_once_with( + args=[self.organization.id], + kwargs={"dry_run": True, "strategy": "agentic_triage", "max_candidates": 3}, + ) + + def test_trigger_clamps_max_candidates(self) -> None: + with patch( + "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org" + ) as mock_task: + response = self.get_success_response( + organization_id=self.organization.id, + max_candidates=500, + status_code=200, + ) + + assert response.data["max_candidates"] == 50 + mock_task.apply_async.assert_called_once() + + def test_trigger_rejects_unknown_strategy(self) -> None: + response = self.get_response( + organization_id=self.organization.id, + strategy="nonexistent_v99", + ) + assert response.status_code == 400 + assert "unknown strategy" in response.data["detail"] + + def test_trigger_rejects_invalid_max_candidates(self) -> None: + response = self.get_response( + organization_id=self.organization.id, + max_candidates="not-a-number", + ) + assert response.status_code == 400 + assert response.data["detail"] == "max_candidates must be a valid integer" + def test_missing_organization_id(self) -> None: response = self.get_response() assert response.status_code == 400 diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index 510104ba06cc..ea444986e080 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -253,11 +253,14 @@ def test_triage_error_records_error_message(self) -> None: project, "fixable", seer_fixability_score=0.9, times_seen=5 ) + def boom(projects, organization, max_candidates): + raise RuntimeError("boom") + with ( self.feature("organizations:seer-project-settings-read-from-sentry"), - patch( - "sentry.tasks.seer.night_shift.cron.agentic_triage_strategy", - side_effect=RuntimeError("boom"), + patch.dict( + "sentry.tasks.seer.night_shift.strategies.TRIAGE_STRATEGIES", + {"agentic_triage": boom}, ), ): run_night_shift_for_org(org.id) @@ -351,9 +354,9 @@ def test_empty_candidates_creates_run_with_no_issues(self) -> None: with ( self.feature("organizations:seer-project-settings-read-from-sentry"), - patch( - "sentry.tasks.seer.night_shift.cron.agentic_triage_strategy", - return_value=([], None), + patch.dict( + "sentry.tasks.seer.night_shift.strategies.TRIAGE_STRATEGIES", + {"agentic_triage": lambda projects, organization, max_candidates: ([], None)}, ), ): run_night_shift_for_org(org.id) @@ -392,7 +395,7 @@ def test_ranks_and_captures_signals(self) -> None: project, f"null-{i}", seer_fixability_score=None, times_seen=100 ) - result = fixability_score_strategy([project]) + result = fixability_score_strategy([project], max_candidates=10) assert result[0].group.id == high.id assert result[0].fixability == 0.9 diff --git a/tests/sentry/tasks/seer/test_night_shift_strategies.py b/tests/sentry/tasks/seer/test_night_shift_strategies.py new file mode 100644 index 000000000000..936d5b18ece9 --- /dev/null +++ b/tests/sentry/tasks/seer/test_night_shift_strategies.py @@ -0,0 +1,37 @@ +from unittest.mock import patch + +from sentry.tasks.seer.night_shift.strategies import ( + DEFAULT_TRIAGE_STRATEGY, + TRIAGE_STRATEGIES, + resolve_triage_strategy, +) +from sentry.testutils.cases import TestCase + + +class ResolveTriageStrategyTest(TestCase): + def test_override_wins(self) -> None: + with self.options({"seer.night_shift.default_strategy": "agentic_triage"}): + name, fn = resolve_triage_strategy("agentic_triage") + assert name == "agentic_triage" + assert fn is TRIAGE_STRATEGIES["agentic_triage"] + + def test_uses_option_when_no_override(self) -> None: + fake = lambda projects, organization, max_candidates: ([], None) + with ( + patch.dict(TRIAGE_STRATEGIES, {"fake_v2": fake}), + self.options({"seer.night_shift.default_strategy": "fake_v2"}), + ): + name, resolved = resolve_triage_strategy(None) + assert name == "fake_v2" + assert resolved is fake + + def test_falls_back_on_unknown_name(self) -> None: + with self.options({"seer.night_shift.default_strategy": "does_not_exist"}): + name, fn = resolve_triage_strategy(None) + assert name == DEFAULT_TRIAGE_STRATEGY + assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] + + def test_falls_back_on_unknown_override(self) -> None: + name, fn = resolve_triage_strategy("bogus_override") + assert name == DEFAULT_TRIAGE_STRATEGY + assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] From abbbdd68011c76d7f7a168e1c13b6d7b8126b2b5 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:12:41 -0400 Subject: [PATCH 2/8] ref(seer): Drop max_candidates clamp on admin night shift endpoint The clamp was defensive against accidental huge batches, but this is a staff-only testing surface where silently rewriting the input is more confusing than helpful. Non-integer values still 400. --- .../seer/endpoints/admin_night_shift_trigger.py | 3 --- .../endpoints/test_admin_night_shift_trigger.py | 13 ------------- 2 files changed, 16 deletions(-) diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index 9c7e920e4b89..b10fc2ddc61e 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -8,8 +8,6 @@ from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org from sentry.tasks.seer.night_shift.strategies import TRIAGE_STRATEGIES -MAX_CANDIDATES_CEILING = 50 - @internal_cell_silo_endpoint class SeerAdminNightShiftTriggerEndpoint(Endpoint): @@ -49,7 +47,6 @@ def post(self, request: Request) -> Response: max_candidates = int(max_candidates_raw) except (ValueError, TypeError): return Response({"detail": "max_candidates must be a valid integer"}, status=400) - max_candidates = max(1, min(max_candidates, MAX_CANDIDATES_CEILING)) run_night_shift_for_org.apply_async( args=[organization_id], diff --git a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py index e588d0badd54..2ff2c2b99527 100644 --- a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py +++ b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py @@ -54,19 +54,6 @@ def test_trigger_with_overrides(self) -> None: kwargs={"dry_run": True, "strategy": "agentic_triage", "max_candidates": 3}, ) - def test_trigger_clamps_max_candidates(self) -> None: - with patch( - "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org" - ) as mock_task: - response = self.get_success_response( - organization_id=self.organization.id, - max_candidates=500, - status_code=200, - ) - - assert response.data["max_candidates"] == 50 - mock_task.apply_async.assert_called_once() - def test_trigger_rejects_unknown_strategy(self) -> None: response = self.get_response( organization_id=self.organization.id, From 780e3f0ae60b2a11d759169519818341f44a3702 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:15:44 -0400 Subject: [PATCH 3/8] ref(seer): Drop premature default_strategy option Swapping the prod default strategy is a code-and-deploy operation in practice; an options-automator knob is overkill for a single-entry registry today. The per-call admin override is enough for testing, and a future strategy change still updates DEFAULT_TRIAGE_STRATEGY. --- src/sentry/options/defaults.py | 6 ---- .../tasks/seer/night_shift/strategies.py | 7 ++--- .../tasks/seer/test_night_shift_strategies.py | 28 +++++-------------- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 4397c0dd0f79..f519d8820543 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -1380,12 +1380,6 @@ default=10, flags=FLAG_AUTOMATOR_MODIFIABLE, ) -register( - "seer.night_shift.default_strategy", - type=String, - default="agentic_triage", - flags=FLAG_AUTOMATOR_MODIFIABLE, -) register( "seer.supergroups_backfill_lightweight.killswitch", diff --git a/src/sentry/tasks/seer/night_shift/strategies.py b/src/sentry/tasks/seer/night_shift/strategies.py index 1a3adc196207..88be7ce3d79b 100644 --- a/src/sentry/tasks/seer/night_shift/strategies.py +++ b/src/sentry/tasks/seer/night_shift/strategies.py @@ -3,7 +3,6 @@ import logging from collections.abc import Callable, Sequence -from sentry import options from sentry.models.organization import Organization from sentry.models.project import Project from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy @@ -25,10 +24,10 @@ def resolve_triage_strategy(override: str | None) -> tuple[str, TriageStrategyFn]: """ - Resolve a strategy name using: explicit override -> options -> constant default. - If the resolved name isn't in the registry, log and fall back to the default. + Resolve a strategy name, falling back to DEFAULT_TRIAGE_STRATEGY when + override is None or points at an unknown strategy. """ - name = override or options.get("seer.night_shift.default_strategy") or DEFAULT_TRIAGE_STRATEGY + name = override or DEFAULT_TRIAGE_STRATEGY fn = TRIAGE_STRATEGIES.get(name) if fn is None: logger.warning( diff --git a/tests/sentry/tasks/seer/test_night_shift_strategies.py b/tests/sentry/tasks/seer/test_night_shift_strategies.py index 936d5b18ece9..93692619626d 100644 --- a/tests/sentry/tasks/seer/test_night_shift_strategies.py +++ b/tests/sentry/tasks/seer/test_night_shift_strategies.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - from sentry.tasks.seer.night_shift.strategies import ( DEFAULT_TRIAGE_STRATEGY, TRIAGE_STRATEGIES, @@ -10,26 +8,14 @@ class ResolveTriageStrategyTest(TestCase): def test_override_wins(self) -> None: - with self.options({"seer.night_shift.default_strategy": "agentic_triage"}): - name, fn = resolve_triage_strategy("agentic_triage") - assert name == "agentic_triage" - assert fn is TRIAGE_STRATEGIES["agentic_triage"] - - def test_uses_option_when_no_override(self) -> None: - fake = lambda projects, organization, max_candidates: ([], None) - with ( - patch.dict(TRIAGE_STRATEGIES, {"fake_v2": fake}), - self.options({"seer.night_shift.default_strategy": "fake_v2"}), - ): - name, resolved = resolve_triage_strategy(None) - assert name == "fake_v2" - assert resolved is fake + name, fn = resolve_triage_strategy("agentic_triage") + assert name == "agentic_triage" + assert fn is TRIAGE_STRATEGIES["agentic_triage"] - def test_falls_back_on_unknown_name(self) -> None: - with self.options({"seer.night_shift.default_strategy": "does_not_exist"}): - name, fn = resolve_triage_strategy(None) - assert name == DEFAULT_TRIAGE_STRATEGY - assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] + def test_no_override_uses_default(self) -> None: + name, fn = resolve_triage_strategy(None) + assert name == DEFAULT_TRIAGE_STRATEGY + assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] def test_falls_back_on_unknown_override(self) -> None: name, fn = resolve_triage_strategy("bogus_override") From a626c6ae559c667d96a812abc2aeabebb5f52019 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:23:15 -0400 Subject: [PATCH 4/8] ref(seer): Introduce TriageStrategyName enum for registry keys StrEnum keeps the registry/default wired to a single source of truth so future strategy additions stop being magic-string edits. Values remain plain strings at every boundary (HTTP, DB, logs). --- src/sentry/tasks/seer/night_shift/strategies.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/strategies.py b/src/sentry/tasks/seer/night_shift/strategies.py index 88be7ce3d79b..07d76cedbdc8 100644 --- a/src/sentry/tasks/seer/night_shift/strategies.py +++ b/src/sentry/tasks/seer/night_shift/strategies.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum import logging from collections.abc import Callable, Sequence @@ -10,16 +11,21 @@ logger = logging.getLogger("sentry.tasks.seer.night_shift") + +class TriageStrategyName(enum.StrEnum): + AGENTIC_TRIAGE = "agentic_triage" + + TriageStrategyFn = Callable[ [Sequence[Project], Organization, int], tuple[list[TriageResult], int | None], ] TRIAGE_STRATEGIES: dict[str, TriageStrategyFn] = { - "agentic_triage": agentic_triage_strategy, + TriageStrategyName.AGENTIC_TRIAGE: agentic_triage_strategy, } -DEFAULT_TRIAGE_STRATEGY = "agentic_triage" +DEFAULT_TRIAGE_STRATEGY = TriageStrategyName.AGENTIC_TRIAGE def resolve_triage_strategy(override: str | None) -> tuple[str, TriageStrategyFn]: From d2afcc74ba974d8771ac1b7b63772719e0619160 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:28:17 -0400 Subject: [PATCH 5/8] ref(seer): Remove premature triage strategy registry With a single strategy today, the registry/enum/resolver is scaffolding for a need that doesn't exist. Revert to calling agentic_triage_strategy directly and keep only the max_candidates admin override, which is the actual testing pain point. Multi-strategy support can be added in the same PR that introduces the second strategy. --- .../endpoints/admin_night_shift_trigger.py | 17 +------ src/sentry/tasks/seer/night_shift/cron.py | 8 ++-- .../tasks/seer/night_shift/strategies.py | 45 ------------------- .../test_admin_night_shift_trigger.py | 17 ++----- tests/sentry/tasks/seer/test_night_shift.py | 15 +++---- .../tasks/seer/test_night_shift_strategies.py | 23 ---------- 6 files changed, 13 insertions(+), 112 deletions(-) delete mode 100644 src/sentry/tasks/seer/night_shift/strategies.py delete mode 100644 tests/sentry/tasks/seer/test_night_shift_strategies.py diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index b10fc2ddc61e..436274681d6b 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -6,7 +6,6 @@ from sentry.api.base import Endpoint, internal_cell_silo_endpoint from sentry.api.permissions import StaffPermission from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org -from sentry.tasks.seer.night_shift.strategies import TRIAGE_STRATEGIES @internal_cell_silo_endpoint @@ -29,15 +28,6 @@ def post(self, request: Request) -> Response: dry_run = bool(request.data.get("dry_run", False)) - strategy = request.data.get("strategy") or None - if strategy is not None and strategy not in TRIAGE_STRATEGIES: - return Response( - { - "detail": f"unknown strategy {strategy!r}; valid options: {sorted(TRIAGE_STRATEGIES)}" - }, - status=400, - ) - max_candidates_raw = request.data.get("max_candidates") max_candidates: int | None if max_candidates_raw in (None, ""): @@ -50,11 +40,7 @@ def post(self, request: Request) -> Response: run_night_shift_for_org.apply_async( args=[organization_id], - kwargs={ - "dry_run": dry_run, - "strategy": strategy, - "max_candidates": max_candidates, - }, + kwargs={"dry_run": dry_run, "max_candidates": max_candidates}, ) return Response( @@ -62,7 +48,6 @@ def post(self, request: Request) -> Response: "success": True, "organization_id": organization_id, "dry_run": dry_run, - "strategy": strategy, "max_candidates": max_candidates, } ) diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index 012330e0b5ac..9b2215fbe00c 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -25,8 +25,8 @@ from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue from sentry.seer.models.seer_api_models import SeerProjectPreference from sentry.tasks.base import instrumented_task +from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy from sentry.tasks.seer.night_shift.models import TriageAction -from sentry.tasks.seer.night_shift.strategies import resolve_triage_strategy from sentry.taskworker.namespaces import seer_tasks from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -90,7 +90,6 @@ def schedule_night_shift() -> None: def run_night_shift_for_org( organization_id: int, dry_run: bool = False, - strategy: str | None = None, max_candidates: int | None = None, ) -> None: try: @@ -131,7 +130,6 @@ def run_night_shift_for_org( sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible_projects)) - triage_strategy, strategy_fn = resolve_triage_strategy(strategy) resolved_max_candidates = ( max_candidates if max_candidates is not None @@ -139,12 +137,12 @@ def run_night_shift_for_org( ) run = SeerNightShiftRun.objects.create( organization=organization, - triage_strategy=triage_strategy, + triage_strategy="agentic_triage", ) agent_run_id = None try: - candidates, agent_run_id = strategy_fn( + candidates, agent_run_id = agentic_triage_strategy( eligible_projects, organization, resolved_max_candidates ) diff --git a/src/sentry/tasks/seer/night_shift/strategies.py b/src/sentry/tasks/seer/night_shift/strategies.py deleted file mode 100644 index 07d76cedbdc8..000000000000 --- a/src/sentry/tasks/seer/night_shift/strategies.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import enum -import logging -from collections.abc import Callable, Sequence - -from sentry.models.organization import Organization -from sentry.models.project import Project -from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy -from sentry.tasks.seer.night_shift.models import TriageResult - -logger = logging.getLogger("sentry.tasks.seer.night_shift") - - -class TriageStrategyName(enum.StrEnum): - AGENTIC_TRIAGE = "agentic_triage" - - -TriageStrategyFn = Callable[ - [Sequence[Project], Organization, int], - tuple[list[TriageResult], int | None], -] - -TRIAGE_STRATEGIES: dict[str, TriageStrategyFn] = { - TriageStrategyName.AGENTIC_TRIAGE: agentic_triage_strategy, -} - -DEFAULT_TRIAGE_STRATEGY = TriageStrategyName.AGENTIC_TRIAGE - - -def resolve_triage_strategy(override: str | None) -> tuple[str, TriageStrategyFn]: - """ - Resolve a strategy name, falling back to DEFAULT_TRIAGE_STRATEGY when - override is None or points at an unknown strategy. - """ - name = override or DEFAULT_TRIAGE_STRATEGY - fn = TRIAGE_STRATEGIES.get(name) - if fn is None: - logger.warning( - "night_shift.unknown_strategy", - extra={"requested": name, "fallback": DEFAULT_TRIAGE_STRATEGY}, - ) - name = DEFAULT_TRIAGE_STRATEGY - fn = TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] - return name, fn diff --git a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py index 2ff2c2b99527..cd4815dbb9f0 100644 --- a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py +++ b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py @@ -28,40 +28,29 @@ def test_trigger_night_shift(self) -> None: assert response.data["success"] is True assert response.data["organization_id"] == self.organization.id - assert response.data["strategy"] is None assert response.data["max_candidates"] is None mock_task.apply_async.assert_called_once_with( args=[self.organization.id], - kwargs={"dry_run": False, "strategy": None, "max_candidates": None}, + kwargs={"dry_run": False, "max_candidates": None}, ) - def test_trigger_with_overrides(self) -> None: + def test_trigger_with_max_candidates_override(self) -> None: with patch( "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org" ) as mock_task: response = self.get_success_response( organization_id=self.organization.id, - strategy="agentic_triage", max_candidates=3, dry_run=True, status_code=200, ) - assert response.data["strategy"] == "agentic_triage" assert response.data["max_candidates"] == 3 mock_task.apply_async.assert_called_once_with( args=[self.organization.id], - kwargs={"dry_run": True, "strategy": "agentic_triage", "max_candidates": 3}, + kwargs={"dry_run": True, "max_candidates": 3}, ) - def test_trigger_rejects_unknown_strategy(self) -> None: - response = self.get_response( - organization_id=self.organization.id, - strategy="nonexistent_v99", - ) - assert response.status_code == 400 - assert "unknown strategy" in response.data["detail"] - def test_trigger_rejects_invalid_max_candidates(self) -> None: response = self.get_response( organization_id=self.organization.id, diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index ea444986e080..5d3aed63c3fb 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -253,14 +253,11 @@ def test_triage_error_records_error_message(self) -> None: project, "fixable", seer_fixability_score=0.9, times_seen=5 ) - def boom(projects, organization, max_candidates): - raise RuntimeError("boom") - with ( self.feature("organizations:seer-project-settings-read-from-sentry"), - patch.dict( - "sentry.tasks.seer.night_shift.strategies.TRIAGE_STRATEGIES", - {"agentic_triage": boom}, + patch( + "sentry.tasks.seer.night_shift.cron.agentic_triage_strategy", + side_effect=RuntimeError("boom"), ), ): run_night_shift_for_org(org.id) @@ -354,9 +351,9 @@ def test_empty_candidates_creates_run_with_no_issues(self) -> None: with ( self.feature("organizations:seer-project-settings-read-from-sentry"), - patch.dict( - "sentry.tasks.seer.night_shift.strategies.TRIAGE_STRATEGIES", - {"agentic_triage": lambda projects, organization, max_candidates: ([], None)}, + patch( + "sentry.tasks.seer.night_shift.cron.agentic_triage_strategy", + return_value=([], None), ), ): run_night_shift_for_org(org.id) diff --git a/tests/sentry/tasks/seer/test_night_shift_strategies.py b/tests/sentry/tasks/seer/test_night_shift_strategies.py deleted file mode 100644 index 93692619626d..000000000000 --- a/tests/sentry/tasks/seer/test_night_shift_strategies.py +++ /dev/null @@ -1,23 +0,0 @@ -from sentry.tasks.seer.night_shift.strategies import ( - DEFAULT_TRIAGE_STRATEGY, - TRIAGE_STRATEGIES, - resolve_triage_strategy, -) -from sentry.testutils.cases import TestCase - - -class ResolveTriageStrategyTest(TestCase): - def test_override_wins(self) -> None: - name, fn = resolve_triage_strategy("agentic_triage") - assert name == "agentic_triage" - assert fn is TRIAGE_STRATEGIES["agentic_triage"] - - def test_no_override_uses_default(self) -> None: - name, fn = resolve_triage_strategy(None) - assert name == DEFAULT_TRIAGE_STRATEGY - assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] - - def test_falls_back_on_unknown_override(self) -> None: - name, fn = resolve_triage_strategy("bogus_override") - assert name == DEFAULT_TRIAGE_STRATEGY - assert fn is TRIAGE_STRATEGIES[DEFAULT_TRIAGE_STRATEGY] From 550d54803644d1cc97b0c348d4604159de4c0b19 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:31:40 -0400 Subject: [PATCH 6/8] feat(seer): Add --max-candidates flag to trigger-night-shift script Mirrors the admin endpoint override so local runs can test with a smaller batch without twiddling the seer.night_shift.issues_per_org option. --- bin/seer/trigger-night-shift | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/seer/trigger-night-shift b/bin/seer/trigger-night-shift index 3c146f8b23ff..82882d28118b 100755 --- a/bin/seer/trigger-night-shift +++ b/bin/seer/trigger-night-shift @@ -10,9 +10,9 @@ import sys from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org -def main(org_id: int) -> None: +def main(org_id: int, max_candidates: int | None) -> None: sys.stdout.write(f"> Running night shift for organization {org_id}...\n") - run_night_shift_for_org(org_id) + run_night_shift_for_org(org_id, max_candidates=max_candidates) sys.stdout.write("> Done.\n") @@ -21,5 +21,11 @@ if __name__ == "__main__": parser.add_argument( "org_id", nargs="?", default=1, type=int, help="Organization ID (default: 1)" ) + parser.add_argument( + "--max-candidates", + type=int, + default=None, + help="Override the candidate cap (default: seer.night_shift.issues_per_org option)", + ) args = parser.parse_args() - main(args.org_id) + main(args.org_id, args.max_candidates) From d512c7f3974ee1d906a2f3f958509a1a1d9eabb1 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:34:01 -0400 Subject: [PATCH 7/8] fix(seer): Reject non-positive max_candidates at entry points Without validation, a negative `max_candidates` hits Python slice semantics in `fixability_score_strategy` (e.g. `candidates[:-1]` returns all but the last), which processes far more candidates than intended. Reject `< 1` at the admin endpoint (400) and in the CLI's argparse type. --- bin/seer/trigger-night-shift | 9 ++++++++- src/sentry/seer/endpoints/admin_night_shift_trigger.py | 2 ++ .../seer/endpoints/test_admin_night_shift_trigger.py | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bin/seer/trigger-night-shift b/bin/seer/trigger-night-shift index 82882d28118b..606d9c7f0def 100755 --- a/bin/seer/trigger-night-shift +++ b/bin/seer/trigger-night-shift @@ -10,6 +10,13 @@ import sys from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError(f"must be >= 1, got {parsed}") + return parsed + + def main(org_id: int, max_candidates: int | None) -> None: sys.stdout.write(f"> Running night shift for organization {org_id}...\n") run_night_shift_for_org(org_id, max_candidates=max_candidates) @@ -23,7 +30,7 @@ if __name__ == "__main__": ) parser.add_argument( "--max-candidates", - type=int, + type=_positive_int, default=None, help="Override the candidate cap (default: seer.night_shift.issues_per_org option)", ) diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index 436274681d6b..843636a50f6d 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -37,6 +37,8 @@ def post(self, request: Request) -> Response: max_candidates = int(max_candidates_raw) except (ValueError, TypeError): return Response({"detail": "max_candidates must be a valid integer"}, status=400) + if max_candidates < 1: + return Response({"detail": "max_candidates must be >= 1"}, status=400) run_night_shift_for_org.apply_async( args=[organization_id], diff --git a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py index cd4815dbb9f0..c155a272f423 100644 --- a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py +++ b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py @@ -59,6 +59,15 @@ def test_trigger_rejects_invalid_max_candidates(self) -> None: assert response.status_code == 400 assert response.data["detail"] == "max_candidates must be a valid integer" + def test_trigger_rejects_non_positive_max_candidates(self) -> None: + for value in (0, -1): + response = self.get_response( + organization_id=self.organization.id, + max_candidates=value, + ) + assert response.status_code == 400, value + assert response.data["detail"] == "max_candidates must be >= 1" + def test_missing_organization_id(self) -> None: response = self.get_response() assert response.status_code == 400 From b2b3cf175466a70a375fba125e938608afc1bcb4 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 16 Apr 2026 16:44:52 -0400 Subject: [PATCH 8/8] fix(seer): Unpack max_candidates check so mypy can narrow None `x in (None, "")` doesn't narrow the type, so int(x) was flagged as `Any | None` incompatible. Split into an explicit `is None` check. --- src/sentry/seer/endpoints/admin_night_shift_trigger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index 843636a50f6d..7bcc9388b5a7 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -30,7 +30,7 @@ def post(self, request: Request) -> Response: max_candidates_raw = request.data.get("max_candidates") max_candidates: int | None - if max_candidates_raw in (None, ""): + if max_candidates_raw is None or max_candidates_raw == "": max_candidates = None else: try: