feat(seer): Allow max candidate override on admin night shift trigger

bin/seer/trigger-night-shift

@@ -10,9 +10,16 @@ import sys
 from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org
 
 
-def main(org_id: int) -> None:
+def _positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed < 1:
+        raise argparse.ArgumentTypeError(f"must be >= 1, got {parsed}")
+    return parsed
+
+
+def main(org_id: int, max_candidates: int | None) -> None:
     sys.stdout.write(f"> Running night shift for organization {org_id}...\n")
-    run_night_shift_for_org(org_id)
+    run_night_shift_for_org(org_id, max_candidates=max_candidates)
     sys.stdout.write("> Done.\n")
 
 
@@ -21,5 +28,11 @@ if __name__ == "__main__":
     parser.add_argument(
         "org_id", nargs="?", default=1, type=int, help="Organization ID (default: 1)"
     )
+    parser.add_argument(
+        "--max-candidates",
+        type=_positive_int,
+        default=None,
+        help="Override the candidate cap (default: seer.night_shift.issues_per_org option)",
+    )
     args = parser.parse_args()
-    main(args.org_id)
+    main(args.org_id, args.max_candidates)

src/sentry/options/defaults.py

@@ -1377,7 +1377,7 @@
 )
 register(
     "seer.night_shift.issues_per_org",
-    default=5,
+    default=10,
     flags=FLAG_AUTOMATOR_MODIFIABLE,
 )
 

src/sentry/seer/endpoints/admin_night_shift_trigger.py

@@ -28,12 +28,28 @@ def post(self, request: Request) -> Response:
 
         dry_run = bool(request.data.get("dry_run", False))
 
-        run_night_shift_for_org.apply_async(args=[organization_id], kwargs={"dry_run": dry_run})
+        max_candidates_raw = request.data.get("max_candidates")
+        max_candidates: int | None
+        if max_candidates_raw is None or max_candidates_raw == "":
+            max_candidates = None
+        else:
+            try:
+                max_candidates = int(max_candidates_raw)
+            except (ValueError, TypeError):
+                return Response({"detail": "max_candidates must be a valid integer"}, status=400)
+            if max_candidates < 1:
+                return Response({"detail": "max_candidates must be >= 1"}, status=400)
+
+        run_night_shift_for_org.apply_async(
+            args=[organization_id],
+            kwargs={"dry_run": dry_run, "max_candidates": max_candidates},
+        )
 
         return Response(
             {
                 "success": True,
                 "organization_id": organization_id,
                 "dry_run": dry_run,
+                "max_candidates": max_candidates,
             }
         )

src/sentry/tasks/seer/night_shift/agentic_triage.py

@@ -35,6 +35,7 @@ class _TriageResponse(pydantic.BaseModel):
 def agentic_triage_strategy(
     projects: Sequence[Project],
     organization: Organization,
+    max_candidates: int,
 ) -> tuple[list[TriageResult], int | None]:
     """
     Select candidates via fixability scoring, then use the Seer Explorer agent
@@ -43,7 +44,7 @@ def agentic_triage_strategy(
     Returns a tuple of (triage_results, agent_run_id).
     """
     # TODO: try a new way to get scored issues
-    scored = fixability_score_strategy(projects)
+    scored = fixability_score_strategy(projects, max_candidates)
     if not scored:
         return [], None
 

src/sentry/tasks/seer/night_shift/cron.py

@@ -87,7 +87,11 @@ def schedule_night_shift() -> None:
     namespace=seer_tasks,
     processing_deadline_duration=5 * 60,
 )
-def run_night_shift_for_org(organization_id: int, dry_run: bool = False) -> None:
+def run_night_shift_for_org(
+    organization_id: int,
+    dry_run: bool = False,
+    max_candidates: int | None = None,
+) -> None:
     try:
         organization = Organization.objects.get(
             id=organization_id, status=OrganizationStatus.ACTIVE
@@ -126,15 +130,21 @@ def run_night_shift_for_org(organization_id: int, dry_run: bool = False) -> None
 
     sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible_projects))
 
-    triage_strategy = "agentic_triage"
+    resolved_max_candidates = (
+        max_candidates
+        if max_candidates is not None
+        else options.get("seer.night_shift.issues_per_org")
+    )
     run = SeerNightShiftRun.objects.create(
         organization=organization,
-        triage_strategy=triage_strategy,
+        triage_strategy="agentic_triage",
     )
 
     agent_run_id = None
     try:
-        candidates, agent_run_id = agentic_triage_strategy(eligible_projects, organization)
+        candidates, agent_run_id = agentic_triage_strategy(
+            eligible_projects, organization, resolved_max_candidates
+        )
 
         if candidates:
             SeerNightShiftRunIssue.objects.bulk_create(

src/sentry/tasks/seer/night_shift/simple_triage.py

@@ -17,7 +17,6 @@
 
 logger = logging.getLogger("sentry.tasks.seer.night_shift")
 
-NIGHT_SHIFT_MAX_CANDIDATES = 10
 NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100
 
 # Weights for candidate scoring. Set to 0 to disable a signal.
@@ -49,6 +48,7 @@ def __lt__(self, other: ScoredCandidate) -> bool:
 
 def fixability_score_strategy(
     projects: Sequence[Project],
+    max_candidates: int,
 ) -> list[ScoredCandidate]:
     """
     Fetch top recommended unresolved issues that haven't been triaged by Seer yet,
@@ -88,7 +88,7 @@ def fixability_score_strategy(
         )
 
     candidates.sort(reverse=True)
-    selected = candidates[:NIGHT_SHIFT_MAX_CANDIDATES]
+    selected = candidates[:max_candidates]
 
     for c in selected:
         sentry_sdk.metrics.distribution("night_shift.fixability_score", c.fixability)

tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py

@@ -28,10 +28,46 @@ def test_trigger_night_shift(self) -> None:
 
         assert response.data["success"] is True
         assert response.data["organization_id"] == self.organization.id
+        assert response.data["max_candidates"] is None
         mock_task.apply_async.assert_called_once_with(
-            args=[self.organization.id], kwargs={"dry_run": False}
+            args=[self.organization.id],
+            kwargs={"dry_run": False, "max_candidates": None},
         )
 
+    def test_trigger_with_max_candidates_override(self) -> None:
+        with patch(
+            "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org"
+        ) as mock_task:
+            response = self.get_success_response(
+                organization_id=self.organization.id,
+                max_candidates=3,
+                dry_run=True,
+                status_code=200,
+            )
+
+        assert response.data["max_candidates"] == 3
+        mock_task.apply_async.assert_called_once_with(
+            args=[self.organization.id],
+            kwargs={"dry_run": True, "max_candidates": 3},
+        )
+
+    def test_trigger_rejects_invalid_max_candidates(self) -> None:
+        response = self.get_response(
+            organization_id=self.organization.id,
+            max_candidates="not-a-number",
+        )
+        assert response.status_code == 400
+        assert response.data["detail"] == "max_candidates must be a valid integer"
+
+    def test_trigger_rejects_non_positive_max_candidates(self) -> None:
+        for value in (0, -1):
+            response = self.get_response(
+                organization_id=self.organization.id,
+                max_candidates=value,
+            )
+            assert response.status_code == 400, value
+            assert response.data["detail"] == "max_candidates must be >= 1"
+
     def test_missing_organization_id(self) -> None:
         response = self.get_response()
         assert response.status_code == 400

tests/sentry/tasks/seer/test_night_shift.py

@@ -392,7 +392,7 @@ def test_ranks_and_captures_signals(self) -> None:
                 project, f"null-{i}", seer_fixability_score=None, times_seen=100
             )
 
-        result = fixability_score_strategy([project])
+        result = fixability_score_strategy([project], max_candidates=10)
 
         assert result[0].group.id == high.id
         assert result[0].fixability == 0.9