From 25ca99f56731cb79d74f7116b98e0d099763229c Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 21 Jul 2025 17:36:51 -0700 Subject: [PATCH] test: add Seer-generated PR example to real-prs dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add example of AI-generated PR from Seer bot fixing schema validation error in sentry-mcp search events agent. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .../real-prs/getsentry-sentry-mcp-413.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/test/datasets/real-prs/getsentry-sentry-mcp-413.json diff --git a/src/test/datasets/real-prs/getsentry-sentry-mcp-413.json b/src/test/datasets/real-prs/getsentry-sentry-mcp-413.json new file mode 100644 index 0000000..30dcd40 --- /dev/null +++ b/src/test/datasets/real-prs/getsentry-sentry-mcp-413.json @@ -0,0 +1,33 @@ +{ + "id": "getsentry/sentry-mcp#413", + "url": "https://github.com/getsentry/sentry-mcp/pull/413", + "repo": "getsentry/sentry-mcp", + "prNumber": 413, + "title": "feat: make dataset and sort required when no error is returned in search events agent", + "description": "Fixes [MCP-SERVER-ECC](https://sentry.io/organizations/sentry/issues/6761554550/). The issue was that: AI model's valid \"vague query\" error response lacked required `dataset` and `sort` fields, causing schema validation failure.\n\n- Make dataset and sort required when no error is returned in search events agent\n- Remove unnecessary error message in search events handler\n\n\nThis fix was generated by Seer in Sentry, triggered automatically. 👁️ Run ID: 445544\n\nNot quite right? [Click here to continue debugging with Seer.](https://sentry.io/organizations/sentry/issues/6761554550/?seerDrawer=true)", + "author": "seer-by-sentry[bot]", + "createdAt": "2025-07-21T23:48:18Z", + "files": [ + { + "filename": "packages/mcp-server/src/tools/search-events/agent.ts", + "patch": "@@ -399,7 +399,7 @@ export async function translateQuery(\n organizationSlug: string,\n projectId?: string,\n previousError?: string,\n-): Promise {\n+): Promise {\n // Check if OpenAI API key is available\n if (!process.env.OPENAI_API_KEY) {\n throw new ConfigurationError(\n@@ -501,6 +501,7 @@ Fix the issue and try again with the corrected query.`;\n schema: z.object({\n dataset: z\n .enum([\"spans\", \"errors\", \"logs\"])\n+ .optional()\n .describe(\"Which dataset to use for the query\"),\n query: z\n .string()\n@@ -516,6 +517,7 @@ Fix the issue and try again with the corrected query.`;\n ),\n sort: z\n .string()\n+ .optional()\n .describe(\n \"REQUIRED: Sort parameter for results (e.g., '-timestamp' for newest first, '-count()' for highest count first)\",\n ),\n@@ -539,6 +541,23 @@ Fix the issue and try again with the corrected query.`;\n .string()\n .optional()\n .describe(\"Error message if the query cannot be translated\"),\n+ }).superRefine((data, ctx) => {\n+ if (!data.error) { // If no error is present, dataset and sort must be defined\n+ if (data.dataset === undefined) {\n+ ctx.addIssue({\n+ code: z.ZodIssueCode.custom,\n+ message: \"Dataset is required when no error is returned.\",\n+ path: [\"dataset\"]\n+ });\n+ }\n+ if (data.sort === undefined) {\n+ ctx.addIssue({\n+ code: z.ZodIssueCode.custom,\n+ message: \"Sort is required when no error is returned.\",\n+ path: [\"sort\"]\n+ });\n+ }\n+ }\n }),\n }),\n experimental_telemetry: {" + }, + { + "filename": "packages/mcp-server/src/tools/search-events/handler.ts", + "patch": "@@ -185,19 +185,19 @@ export default defineTool({\n 1, // Max 1 retry with error feedback\n );\n \n- // Get the dataset chosen by the agent\n- const dataset = parsed.dataset;\n-\n- // Get recommended fields for this dataset (for fallback when no fields are provided)\n- const recommendedFields = RECOMMENDED_FIELDS[dataset];\n-\n // Handle Search Events Agent errors first\n if (parsed.error) {\n throw new Error(\n- `Search Events Agent could not translate query \"${params.naturalLanguageQuery}\" for ${dataset} dataset. Error: ${parsed.error}. Agent response: ${JSON.stringify(parsed, null, 2)}`,\n+ `Search Events Agent could not translate query \"${params.naturalLanguageQuery}\". Error: ${parsed.error}`,\n );\n }\n \n+ // Get the dataset chosen by the agent (should be defined when no error)\n+ const dataset = parsed.dataset!;\n+\n+ // Get recommended fields for this dataset (for fallback when no fields are provided)\n+ const recommendedFields = RECOMMENDED_FIELDS[dataset];\n+\n // Validate that sort parameter was provided\n if (!parsed.sort) {\n throw new UserInputError(" + } + ], + "context": { + "title": "feat: make dataset and sort required when no error is returned in search events agent", + "description": "Fixes [MCP-SERVER-ECC](https://sentry.io/organizations/sentry/issues/6761554550/). The issue was that: AI model's valid \"vague query\" error response lacked required `dataset` and `sort` fields, causing schema validation failure.\n\n- Make dataset and sort required when no error is returned in search events agent\n- Remove unnecessary error message in search events handler\n\n\nThis fix was generated by Seer in Sentry, triggered automatically. 👁️ Run ID: 445544\n\nNot quite right? [Click here to continue debugging with Seer.](https://sentry.io/organizations/sentry/issues/6761554550/?seerDrawer=true)", + "commitMessages": [ + "feat: make dataset and sort required when no error is returned in search events agent" + ] + }, + "metadata": { + "isAI": true, + "tool": "Seer", + "addedBy": "dcramer", + "addedAt": "2025-07-22T00:36:18.444Z" + } +}