Skip to content

CHORE: Refactoring AI triaging #536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sumitmsft wants to merge 3 commits into
base: main
Choose a base branch
from
+569 −6
Open

CHORE: Refactoring AI triaging #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f07fba7
Add suggested customer response, similar issue detection, and workaro…
sumitmsft Apr 23, 2026
334ea7b
Fix: HTML-escape all LLM-provided fields and validate issue_number in…
sumitmsft Apr 23, 2026
f1066e9
Fix: Validate workaround confidence against allowlist (high|medium|low)
sumitmsft Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 76 additions & 1 deletion .github/workflows/issue-notify.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,18 @@ on:
required: false
type: string
default: ''
suggested_response:
required: false
type: string
default: ''
similar_issues:
required: false
type: string
default: ''
workaround:
required: false
type: string
default: ''
issue_number:
required: true
type: string
Expand Down Expand Up @@ -69,6 +81,9 @@ jobs:
INPUT_SUMMARY: ${{ inputs.summary_for_maintainers }}
INPUT_CODE_ANALYSIS: ${{ inputs.code_analysis }}
INPUT_ENGINEER_GUIDANCE: ${{ inputs.engineer_guidance }}
INPUT_SUGGESTED_RESPONSE: ${{ inputs.suggested_response }}
INPUT_SIMILAR_ISSUES: ${{ inputs.similar_issues }}
INPUT_WORKAROUND: ${{ inputs.workaround }}
INPUT_JUSTIFICATION: ${{ inputs.justification }}
TEAMS_WEBHOOK_URL: ${{ secrets.TEAMS_WEBHOOK_URL }}
run: |
Expand Down Expand Up @@ -142,7 +157,6 @@ jobs:
else empty end),
(if .implementation_approach then "<b>Implementation Approach:</b> " + (.implementation_approach | @html) else empty end),
(if .risks_and_tradeoffs then "<b>Risks &amp; Tradeoffs:</b> " + (.risks_and_tradeoffs | @html) else empty end),
(if .suggested_response then "<b>Suggested Response to User:</b><br>" + (.suggested_response | @html) else empty end),
(if .related_considerations and (.related_considerations | length) > 0
then "<b>Related Considerations:</b><br>" + ([.related_considerations | to_entries[] | "&nbsp;&nbsp;" + ((.key + 1) | tostring) + ". " + (.value | @html)] | join("<br>"))
else empty end)
Expand All @@ -152,6 +166,50 @@ jobs:
ENGINEER_GUIDANCE=""
fi

# Format suggested customer response
SUGGESTED_RESPONSE_RAW="$INPUT_SUGGESTED_RESPONSE"
if [ -n "$SUGGESTED_RESPONSE_RAW" ]; then
SUGGESTED_RESPONSE=$(echo "$SUGGESTED_RESPONSE_RAW" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/$/\<br\>/g')
else
SUGGESTED_RESPONSE=""
fi

# Parse and format similar issues analysis
SIMILAR_ISSUES_RAW="$INPUT_SIMILAR_ISSUES"
if [ -n "$SIMILAR_ISSUES_RAW" ]; then
SIMILAR_ISSUES=$(echo "$SIMILAR_ISSUES_RAW" | jq -r '
[
(if .summary then "<b>Summary:</b> " + (.summary | @html) else empty end),
(if .duplicate_issues and (.duplicate_issues | length) > 0
then "<b>Similar/Duplicate Issues:</b><br>" + ([.duplicate_issues[] | select(.issue_number | tostring | test("^[0-9]+$")) | "&nbsp;&nbsp;• <a href=\"https://github.com/microsoft/mssql-python/issues/" + (.issue_number | tostring) + "\">#" + (.issue_number | tostring) + "</a> " + (.title | @html) + " [" + (.state | @html) + "] — <i>" + (.similarity | @html) + ":</i> " + (.explanation | @html)] | join("<br>"))
else empty end),
(if .recently_fixed and (.recently_fixed | length) > 0
then "<b>Recently Fixed:</b><br>" + ([.recently_fixed[] | select(.issue_number | tostring | test("^[0-9]+$")) | "&nbsp;&nbsp;• <a href=\"https://github.com/microsoft/mssql-python/issues/" + (.issue_number | tostring) + "\">#" + (.issue_number | tostring) + "</a> " + (.title | @html) + " (closed " + (.closed_at | @html) + ") — " + (.relevance | @html)] | join("<br>"))
else empty end)
] | join("<br><br>")
' 2>/dev/null || echo "$SIMILAR_ISSUES_RAW" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
else
SIMILAR_ISSUES=""
fi

# Parse and format workaround analysis
WORKAROUND_RAW="$INPUT_WORKAROUND"
if [ -n "$WORKAROUND_RAW" ]; then
WORKAROUND=$(echo "$WORKAROUND_RAW" | jq -r '
[
(if .summary then "<b>Summary:</b> " + (.summary | @html) else empty end),
(if .has_workaround and .workarounds and (.workarounds | length) > 0
then "<b>Workarounds:</b><br>" + ([.workarounds | to_entries[] | "&nbsp;&nbsp;" + ((.key + 1) | tostring) + ". <b>" + (.value.description | @html) + "</b> [" + (if (.value.confidence | tostring) == "high" or (.value.confidence | tostring) == "medium" or (.value.confidence | tostring) == "low" then (.value.confidence | tostring) else "unknown" end) + " confidence]<br>&nbsp;&nbsp;&nbsp;&nbsp;Limitations: " + (.value.limitations | @html) + (if .value.code_snippet and (.value.code_snippet | length) > 0 then "<br>&nbsp;&nbsp;&nbsp;&nbsp;Code: <code>" + (.value.code_snippet | @html) + "</code>" else "" end)] | join("<br>"))
else empty end),
(if .can_downgrade == true and .downgrade_version
then "⬇️ <b>Safe to downgrade to:</b> " + (.downgrade_version | @html)
else empty end)
Comment thread
sumitmsft marked this conversation as resolved.
] | join("<br><br>")
' 2>/dev/null || echo "$WORKAROUND_RAW" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
else
WORKAROUND=""
fi

# Set severity color indicator
case "$SEVERITY" in
critical) SEV_INDICATOR="🔴" ;;
Expand All @@ -176,6 +234,9 @@ jobs:
--arg summary "$INPUT_SUMMARY" \
--arg code_analysis "$CODE_ANALYSIS" \
--arg engineer_guidance "$ENGINEER_GUIDANCE" \
--arg suggested_response "$SUGGESTED_RESPONSE" \
--arg similar_issues "$SIMILAR_ISSUES" \
--arg workaround "$WORKAROUND" \
--arg justification "$INPUT_JUSTIFICATION" \
--arg action "$ACTION" \
--arg repo_url "https://github.com/microsoft/mssql-python" \
Expand All @@ -198,10 +259,24 @@ jobs:
"<p>" + ($summary | @html) + "</p>" +
"<h3>🔍 Code Analysis</h3>" +
"<p>" + $code_analysis + "</p>" +
(if $similar_issues != "" then
"<h3>🔄 Similar Issues &amp; Recent Fixes</h3>" +
"<p>" + $similar_issues + "</p>"
else "" end) +
(if $workaround != "" then
"<h3>🛠️ Workarounds</h3>" +
"<p>" + $workaround + "</p>"
else "" end) +
(if $engineer_guidance != "" then
"<h3>💡 Engineer Guidance</h3>" +
"<p>" + $engineer_guidance + "</p>"
else "" end) +
(if $suggested_response != "" then
"<hr>" +
"<h3>✉️ Suggested Response to Customer</h3>" +
"<p><i>Copy-paste or edit the response below and post it on the issue:</i></p>" +
"<blockquote>" + $suggested_response + "</blockquote>"
else "" end) +
"<hr>" +
"<p>⚡ <b>Action Required:</b> " + $action + "</p>" +
"<p><i>⚠️ AI-generated analysis — verified against source code but may contain inaccuracies. Review before acting.</i></p>" +
Expand Down
228 changes: 227 additions & 1 deletion .github/workflows/issue-triage.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,6 @@ jobs:
"implementation_approach": "<concrete implementation steps referencing specific functions/lines from the code — ONLY if issue_identified is true, otherwise empty string>",
"effort_estimate": "small|medium|large|epic",
"risks_and_tradeoffs": "<potential risks, backward compatibility concerns, or tradeoffs — ONLY if issue_identified is true, otherwise empty string>",
"suggested_response": "<a draft response the engineer could post on the issue. Always ask the user to share a minimal repro or code snippet that demonstrates the issue or desired behavior, if they haven't already provided one.>",
"related_considerations": ["<other things the team should think about — ONLY if issue_identified is true, otherwise empty array>"]
}

Expand All @@ -243,9 +242,227 @@ jobs:
}
}

// --- Search for similar/duplicate issues and recent fixes ---
console.log('Searching for similar issues and recent fixes...');
let similarIssuesAnalysis = '';

try {
// Fetch recent open issues (last 30)
const openIssues = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
per_page: 30,
sort: 'created',
direction: 'desc'
});

// Fetch recently closed issues (last 30)
const closedIssues = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'closed',
per_page: 30,
sort: 'updated',
direction: 'desc'
});

const allIssues = [...openIssues.data, ...closedIssues.data]
.filter(i => i.number !== issueNumber && !i.pull_request)
.map(i => ({
number: i.number,
title: i.title,
state: i.state,
labels: i.labels.map(l => l.name).join(', '),
created_at: i.created_at,
closed_at: i.closed_at,
body_preview: (i.body || '').slice(0, 300)
}));

console.log(`Found ${allIssues.length} issues to compare against`);

const similarPrompt = `
You are an expert triage system for the mssql-python repository.
A new issue has been filed. Your job is to check if any similar or duplicate issues
already exist (open or recently closed/fixed).

NEW ISSUE:
Title: ${issueTitle}
Body: ${issueBody.slice(0, 2000)}
Keywords: ${analysis.keywords.join(', ')}

EXISTING ISSUES (recent open + recently closed):
${JSON.stringify(allIssues, null, 2).slice(0, 6000)}

Analyze and respond in JSON:
{
"has_duplicates": true/false,
"has_similar": true/false,
"duplicate_issues": [
{
"issue_number": <number>,
"title": "<title>",
"state": "open|closed",
"similarity": "duplicate|highly_similar|related",
"explanation": "<why this is a duplicate/similar>"
}
],
"recently_fixed": [
{
"issue_number": <number>,
"title": "<title>",
"closed_at": "<date>",
"relevance": "<how this fix relates to the new issue>"
}
],
"summary": "<1-3 sentence summary: are there duplicates? was this recently fixed? should the new issue be closed as duplicate?>"
}

IMPORTANT:
- Only flag as duplicate if the core problem is truly the same, not just topically related
- For recently_fixed, only include issues that were closed AND directly address the same problem
- If no duplicates or similar issues exist, set has_duplicates and has_similar to false with empty arrays
`;

const similarResult = await callGitHubModels(similarPrompt);
similarIssuesAnalysis = similarResult;
const parsed = JSON.parse(similarResult);
console.log(`Similar issues check: duplicates=${parsed.has_duplicates}, similar=${parsed.has_similar}`);
} catch (e) {
console.log(`Similar issues search failed: ${e.message}`);
similarIssuesAnalysis = '';
}

// --- Generate workaround suggestions (BUG/BREAK_FIX only) ---
let workaroundAnalysis = '';

if (['BUG', 'BREAK_FIX'].includes(analysis.category)) {
console.log('Generating workaround suggestions...');

const workaroundContext = codeAnalysis
? `Code Analysis:\n${codeAnalysis}`
: '';

const workaroundPrompt = `
You are a senior support engineer on the mssql-python team — Microsoft's Python driver for SQL Server.
A user has reported a ${analysis.category === 'BREAK_FIX' ? 'regression/break-fix' : 'bug'}.
Your job is to suggest practical workarounds the user can apply RIGHT NOW while the team works on a proper fix.

Issue Title: ${issueTitle}
Issue Body:
${issueBody.slice(0, 2000)}

${workaroundContext}
${codeContext}

Provide workaround suggestions in JSON:
{
"has_workaround": true/false,
"workarounds": [
{
"description": "<clear description of the workaround>",
"code_snippet": "<Python code snippet showing the workaround, if applicable>",
"limitations": "<any limitations or caveats of this workaround>",
"confidence": "high|medium|low"
}
],
"can_downgrade": true/false,
"downgrade_version": "<safe version to downgrade to, if applicable>",
"summary": "<1-2 sentence summary of available workarounds>"
}

IMPORTANT:
- Only suggest workarounds you are confident about based on the code and issue description
- Workarounds should be practical and safe for production use
- If no reliable workaround exists, set has_workaround to false
- For BREAK_FIX issues, always consider if downgrading to a previous version is viable
- Code snippets should be complete and copy-pasteable
`;

try {
workaroundAnalysis = await callGitHubModels(workaroundPrompt);
const parsed = JSON.parse(workaroundAnalysis);
console.log(`Workaround analysis: has_workaround=${parsed.has_workaround}`);
} catch (e) {
console.log(`Workaround generation failed: ${e.message}`);
workaroundAnalysis = '';
}
}

// NO labels modified on the issue — label info sent to Teams only
// NO comment posted to the issue

// --- Generate suggested customer response (all categories) ---
console.log('Generating suggested customer response...');
let suggestedResponse = '';

const analysisContext = codeAnalysis
? `Code Analysis:\n${codeAnalysis}`
: engineerGuidance
? `Engineer Guidance:\n${engineerGuidance}`
: '';

const workaroundContext = workaroundAnalysis
? `Workaround Analysis:\n${workaroundAnalysis}`
: '';

const similarContext = similarIssuesAnalysis
? `Similar Issues Analysis:\n${similarIssuesAnalysis}`
: '';

const responsePrompt = `
You are a senior support engineer on the mssql-python team — Microsoft's Python driver for SQL Server.
A customer filed a GitHub issue and you need to craft a helpful, professional, and empathetic response
that an engineer can copy-paste (or lightly edit) and post on the issue.

Issue Category: ${analysis.category}
Severity: ${analysis.severity}
Issue Title: ${issueTitle}
Issue Author: @${issueAuthor}
Issue Body:
${issueBody.slice(0, 3000)}

${analysisContext}

${workaroundContext}

${similarContext}

Write a suggested response following these guidelines:
- Address the author by their GitHub username (@username)
- Thank them for filing the issue
- Acknowledge the specific problem or request they described
- For BUG/BREAK_FIX: Let them know the team is investigating; ask for OS, Python version,
mssql-python version, SQL Server version, and a minimal repro script if not already provided;
if workarounds are available, mention them briefly
- For FEATURE_REQUEST: Acknowledge the value of the request; mention the team will evaluate it;
ask for use-case details or code examples showing desired behavior if not provided
- For DISCUSSION: Provide helpful guidance or clarification based on the analysis;
point to relevant docs or code if applicable
- If similar or duplicate issues were found, mention them (e.g., "This looks related to #123")
- Always ask for a minimal reproduction script/code snippet if the user hasn't provided one
- Keep the tone warm, professional, and collaborative
- Use Markdown formatting suitable for GitHub comments
- Do NOT promise timelines or specific fixes
- Do NOT reveal internal triage details or AI involvement
- Keep it concise (under 200 words)

Respond in JSON:
{
"suggested_response": "<the full Markdown response ready to post on GitHub>"
}
`;

try {
const responseResult = await callGitHubModels(responsePrompt);
const parsed = JSON.parse(responseResult);
suggestedResponse = parsed.suggested_response || '';
console.log('Suggested customer response generated');
} catch (e) {
console.log(`Suggested response generation failed: ${e.message}`);
suggestedResponse = '';
}

// --- Store outputs ---
core.setOutput('category', analysis.category);
core.setOutput('confidence', analysis.confidence.toString());
Expand All @@ -256,6 +473,9 @@ jobs:
core.setOutput('keywords', analysis.keywords.join(', '));
core.setOutput('code_analysis', codeAnalysis);
core.setOutput('engineer_guidance', engineerGuidance);
core.setOutput('suggested_response', suggestedResponse);
core.setOutput('similar_issues', similarIssuesAnalysis);
core.setOutput('workaround', workaroundAnalysis);
core.setOutput('issue_number', issueNumber.toString());
core.setOutput('issue_title', issueTitle);
core.setOutput('issue_url', issue.html_url);
Expand All @@ -271,6 +491,9 @@ jobs:
keywords: ${{ steps.triage.outputs.keywords }}
code_analysis: ${{ steps.triage.outputs.code_analysis }}
engineer_guidance: ${{ steps.triage.outputs.engineer_guidance }}
suggested_response: ${{ steps.triage.outputs.suggested_response }}
similar_issues: ${{ steps.triage.outputs.similar_issues }}
workaround: ${{ steps.triage.outputs.workaround }}
issue_number: ${{ steps.triage.outputs.issue_number }}
issue_title: ${{ steps.triage.outputs.issue_title }}
issue_url: ${{ steps.triage.outputs.issue_url }}
Expand All @@ -289,6 +512,9 @@ jobs:
keywords: ${{ needs.triage.outputs.keywords }}
code_analysis: ${{ needs.triage.outputs.code_analysis }}
engineer_guidance: ${{ needs.triage.outputs.engineer_guidance }}
suggested_response: ${{ needs.triage.outputs.suggested_response }}
similar_issues: ${{ needs.triage.outputs.similar_issues }}
workaround: ${{ needs.triage.outputs.workaround }}
issue_number: ${{ needs.triage.outputs.issue_number }}
issue_title: ${{ needs.triage.outputs.issue_title }}
issue_url: ${{ needs.triage.outputs.issue_url }}
Expand Down
Loading
Loading