From a3e39eae73794d6ed63469f956b6b189d6e750d6 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 29 Oct 2025 12:27:46 +0000 Subject: [PATCH] feat: support configuring chat context in AI assistant This is important for reports with many apps, or many checks. The model limits can be easily hit. --- report-app/report-server.ts | 39 +- .../pages/report-viewer/report-viewer.html | 503 +++++++++--------- .../app/pages/report-viewer/report-viewer.ts | 17 +- .../src/app/services/reports-fetcher.ts | 10 +- .../app/shared/ai-assistant/ai-assistant.html | 47 +- .../app/shared/ai-assistant/ai-assistant.scss | 71 ++- .../app/shared/ai-assistant/ai-assistant.ts | 23 + runner/reporting/report-ai-chat.ts | 132 +++-- runner/reporting/report-ai-summary.ts | 12 +- runner/shared-interfaces.ts | 29 + 10 files changed, 562 insertions(+), 321 deletions(-) diff --git a/report-app/report-server.ts b/report-app/report-server.ts index 4bcd0e6..7525507 100644 --- a/report-app/report-server.ts +++ b/report-app/report-server.ts @@ -10,7 +10,14 @@ import {fileURLToPath} from 'node:url'; import {chatWithReportAI} from '../runner/reporting/report-ai-chat'; import {convertV2ReportToV3Report} from '../runner/reporting/migrations/v2_to_v3'; import {FetchedLocalReports, fetchReportsFromDisk} from '../runner/reporting/report-local-disk'; -import {AiChatRequest, AIConfigState, RunInfo} from '../runner/shared-interfaces'; +import { + AiChatRequest, + AIConfigState, + AssessmentResultFromReportServer, + IndividualAssessmentState, + RunInfo, + RunInfoFromReportServer, +} from '../runner/shared-interfaces'; // This will result in a lot of loading and would slow down the serving, // so it's loaded lazily below. @@ -41,7 +48,7 @@ app.get('/api/reports', async (_, res) => { res.json(results); }); -async function fetchAndMigrateReports(id: string): Promise { +async function fetchAndMigrateReports(id: string): Promise { const localData = await resolveLocalData(options.reportsRoot); let result: RunInfo[] | null = null; @@ -55,8 +62,23 @@ async function fetchAndMigrateReports(id: string): Promise { return null; } - // Convert potential older v2 reports. - return result.map(r => convertV2ReportToV3Report(r)); + let checkID = 0; + return result.map(run => { + const newRun = { + // Convert potential older v2 reports. + ...convertV2ReportToV3Report(run), + // Augment the `RunInfo` to include IDs for individual apps. + // This is useful for the AI chat and context filtering. + results: run.results.map( + check => + ({ + id: `${id}-${checkID++}`, + ...check, + }) satisfies AssessmentResultFromReportServer, + ), + }; + return newRun satisfies RunInfoFromReportServer; + }); } // Endpoint for fetching a specific report group. @@ -89,16 +111,19 @@ app.post('/api/reports/:id/chat', async (req, res) => { } try { - const {prompt, pastMessages, model} = req.body as AiChatRequest; - const assessments = reports.flatMap(run => run.results); + const {prompt, pastMessages, model, contextFilters, openAppIDs} = req.body as AiChatRequest; + const allAssessments = reports.flatMap(run => run.results); + const abortController = new AbortController(); const summary = await chatWithReportAI( await (llm ?? getOrCreateGenkitLlmRunner()), prompt, abortController.signal, - assessments, + allAssessments, pastMessages, model, + contextFilters, + openAppIDs, ); res.json(summary); } catch (e) { diff --git a/report-app/src/app/pages/report-viewer/report-viewer.html b/report-app/src/app/pages/report-viewer/report-viewer.html index b1693e4..9b65958 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.html +++ b/report-app/src/app/pages/report-viewer/report-viewer.html @@ -265,6 +265,7 @@

Logs

} @@ -280,7 +281,11 @@

Generated applications

}
@for (result of filteredResults(); track result) { - + Generated applications
- @if (appPanel.opened()) { -
-
-

Prompt

- 
{{ result.promptDef.prompt }}
-
+
+
+

Prompt

+ 
{{ result.promptDef.prompt }}
+
-
-

Results

- @for (category of result.score.categories; track category.id) { -
- {{ category.name }} ({{ category.points }}/{{ +
+

Results

+ @for (category of result.score.categories; track category.id) { +
+ {{ category.name }} ({{ category.points }}/{{ category.maxPoints - }} - points) -
- -
- @for (check of category.assessments; track check.id) { -
- @if (isSkippedAssessment(check)) { - ➖ - {{ check.name }} - Skipped: {{ check.message }} - } @else { - @let isMax = check.successPercentage === 1; - {{ isMax ? '✔' : '✘' }} - {{ check.name }} - {{ check.message }} - } -
- } @empty { -
No checks
- } -
- } - -
- @let totalPercent = - formatScore( + }} + points) +
+ +
+ @for (check of category.assessments; track check.id) { +
+ @if (isSkippedAssessment(check)) { + ➖ + {{ check.name }} + Skipped: {{ check.message }} + } @else { + @let isMax = check.successPercentage === 1; + {{ isMax ? '✔' : '✘' }} + {{ check.name }} + {{ check.message }} + } +
+ } @empty { +
No checks
+ } +
+ } + +
+ @let totalPercent = + formatScore( result.score.totalPoints, result.score.maxOverallPoints ); - - Total:   - - {{ result.score.totalPoints }} / {{ result.score.maxOverallPoints }} points ({{totalPercent - }}%) - -
+ + Total:   + + {{ result.score.totalPoints }} / {{ result.score.maxOverallPoints }} points ({{totalPercent + }}%) +
+
- @if (result.testResult) { -
-

Test Results

-
- @if (result.testResult.passed) { - ✔ Tests passed - @if ((result.testRepairAttempts || 0) > 0) { -  after {{ result.testRepairAttempts }} repair attempt(s) - } - } @else { - ✘ Tests failed + @if (result.testResult) { +
+

Test Results

+
+ @if (result.testResult.passed) { + ✔ Tests passed + @if ((result.testRepairAttempts || 0) > 0) { +  after {{ result.testRepairAttempts }} repair attempt(s) } -
- - @if (result.testResult.output && !result.testResult.passed) { -
- See Test Output - 
{{ result.testResult.output }}
-
+ } @else { + ✘ Tests failed }
- } -
-

Additional info

- @for (attempt of result.attemptDetails; track attempt) { - @let isBuilt = attempt.buildResult.status === 'success'; - @let axeViolations = attempt.serveTestingResult?.axeViolations; - @let hasAxeViolations = axeViolations && axeViolations.length > 0; - @let testsFailed = attempt.testResult?.passed === false; - - - - {{ + @if (result.testResult.output && !result.testResult.passed) { +
+ See Test Output + 
{{ result.testResult.output }}
+
+ } +
+ } + +
+

Additional info

+ @for (attempt of result.attemptDetails; track attempt) { + @let isBuilt = attempt.buildResult.status === 'success'; + @let axeViolations = attempt.serveTestingResult?.axeViolations; + @let hasAxeViolations = axeViolations && axeViolations.length > 0; + @let testsFailed = attempt.testResult?.passed === false; + + + + {{ $index === 0 ? 'Initial response' : `Repair attempt #${$index}` - }} + }} + + Build + @if (isBuilt) { BuildA11y + } - @if (isBuilt) { - A11y - } - - @if (attempt.testResult) { - Tests - } - + @if (attempt.testResult) { + Tests + } + - @if (expansionPanel.opened()) { - @if (attempt.reasoning) { -
- See LLM Thoughts - 
{{
+                  @if (expansionPanel.opened()) {
+                    @if (attempt.reasoning) {
+                      

+                        See LLM Thoughts
+                        
{{
                             attempt.reasoning
-                          }}

-                        

-                      }
-                      @if (!isBuilt) {
-                        
Build Message

-                        
{{
-                          attempt.buildResult.message
                         }}

-                      }
+
+ } + @if (!isBuilt) { +

Build Message

+ 
{{
+                          attempt.buildResult.message
+                      }}
+ } - @if (hasAxeViolations) { -

A11y Violations

- 
+                    @if (hasAxeViolations) {
+                      
A11y Violations

+                      
                           
    
                             @for (violation of axeViolations; track violation.id) {
                               
  • 
@@ -491,141 +495,140 @@ 
    A11y Violations
    
                             }
                           

                         

-                      }
-
-                      @if (testsFailed) {
-                        
Failed Tests

-                        
{{ attempt.testResult?.output }}

-                      }
-
-                      
Generated Code

-
-                      @for (file of attempt.outputFiles; track file) {
-                        {{ file.filePath }}
-                        

-                          
-                          
-                        

-                        
-                      }
                     }
-                  
-                }
-
-                @let lighthouse = result.finalAttempt.serveTestingResult?.lighthouseResult;
 
-                @if (lighthouse) {
-                  
-                    Lighthouse
-
-                    @for (category of lighthouse.categories; track category.id) {
-                      
+                    @if (testsFailed) {
+                      
Failed Tests

+                      
{{ attempt.testResult?.output }}

                     }
 
-                    @if (lighthouse.uncategorized.length > 0) {
-                      
+                    
Generated Code

+
+                    @for (file of attempt.outputFiles; track file) {
+                      {{ file.filePath }}
+                      

+                        
+                        
+                      

+                      
                     }
-                  
-                }
+                  }
+                
+              }
 
-                @if (result.userJourneys && result.userJourneys.result.length > 0) {
-                  
-                    User Journeys
-                    @for (journey of result.userJourneys.result; track journey.name) {
-                      
{{ journey.name }}

-
-                      
    
-                        @for (step of journey.steps; track $index) {
-                          
  1. {{ step }}
    2. 
-                        }
-                      

-                    }
-                  
-                }
-
+ @let lighthouse = result.finalAttempt.serveTestingResult?.lighthouseResult; -
-

Debugging Tools

+ @if (lighthouse) { + + Lighthouse - + @for (category of lighthouse.categories; track category.id) { + + } - @let debugCommand = getDebugCommand(report, result); + @if (lighthouse.uncategorized.length > 0) { + + } + + } - @if (debugCommand) { -

To see the app locally, run the following command:

- 
{{debugCommand}}
- } + @if (result.userJourneys && result.userJourneys.result.length > 0) { + + User Journeys + @for (journey of result.userJourneys.result; track journey.name) { +

{{ journey.name }}

- @if (result.toolLogs && result.toolLogs.length > 0) { - - Tool Logs -
    - @for (log of result.toolLogs; track $index) { -
  • -
    - @let name = log.request.name; - - Log Entry #{{ $index + 1}}{{ name ? ' - ' + name : '' }} - -
    -
    Request
    - -
    Response
    - -
    -
    -
    • - } @empty { -
  • No MCP logs were recorded for this run.
    • +
      + @for (step of journey.steps; track $index) { +
    1. {{ step }}
      2. } -
- - } -
- - @let finalRuntimeErrors = finalAttempt.serveTestingResult?.runtimeErrors; - @if (finalRuntimeErrors) { -
-

Runtime errors

- 
{{ finalRuntimeErrors }}
-
+ + } + } +
- @let screenshot = getScreenshotUrl(result); +
+

Debugging Tools

- @if (screenshot) { -
-

Screenshot

- -
+ + + @let debugCommand = getDebugCommand(report, result); + + @if (debugCommand) { +

To see the app locally, run the following command:

+ 
{{debugCommand}}
+ } + + @if (result.toolLogs && result.toolLogs.length > 0) { + + Tool Logs +
    + @for (log of result.toolLogs; track $index) { +
  • +
    + @let name = log.request.name; + + Log Entry #{{ $index + 1}}{{ name ? ' - ' + name : '' }} + +
    +
    Request
    + +
    Response
    + +
    +
    +
    • + } @empty { +
  • No MCP logs were recorded for this run.
    • + } +
+ }
- } + + @let finalRuntimeErrors = finalAttempt.serveTestingResult?.runtimeErrors; + @if (finalRuntimeErrors) { +
+

Runtime errors

+ 
{{ finalRuntimeErrors }}
+
+ } + + @let screenshot = getScreenshotUrl(result); + + @if (screenshot) { +
+

Screenshot

+ +
+ } +
}
diff --git a/report-app/src/app/pages/report-viewer/report-viewer.ts b/report-app/src/app/pages/report-viewer/report-viewer.ts index 32c5cc0..f09f7a1 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.ts +++ b/report-app/src/app/pages/report-viewer/report-viewer.ts @@ -16,10 +16,12 @@ import {NgxJsonViewerModule} from 'ngx-json-viewer'; import {BuildErrorType} from '../../../../../runner/workers/builder/builder-types'; import { AssessmentResult, + AssessmentResultFromReportServer, IndividualAssessment, IndividualAssessmentState, LlmResponseFile, RunInfo, + RunInfoFromReportServer, RunSummaryBuilds, RunSummaryTests, RuntimeStats, @@ -81,13 +83,26 @@ export class ReportViewer { protected formatScore = formatScore; protected error = computed(() => this.selectedReport.error()); protected isAiAssistantVisible = signal(false); + protected openAppIDs = signal([]); + + isAppOpen(result: AssessmentResultFromReportServer): boolean { + return this.openAppIDs().includes(result.id); + } + + setAppOpen(result: AssessmentResultFromReportServer, isOpen: boolean): void { + if (isOpen) { + this.openAppIDs.update(ids => [...ids, result.id]); + } else { + this.openAppIDs.update(ids => ids.filter(i => i !== result.id)); + } + } private selectedReport = resource({ params: () => ({groupId: this.reportGroupId()}), loader: ({params}) => this.reportsFetcher.getCombinedReport(params.groupId), }); - protected selectedReportWithSortedResults = computed(() => { + protected selectedReportWithSortedResults = computed(() => { if (!this.selectedReport.hasValue()) { return null; } diff --git a/report-app/src/app/services/reports-fetcher.ts b/report-app/src/app/services/reports-fetcher.ts index 6048bf7..d700c03 100644 --- a/report-app/src/app/services/reports-fetcher.ts +++ b/report-app/src/app/services/reports-fetcher.ts @@ -1,12 +1,12 @@ import {computed, inject, Injectable, PLATFORM_ID, resource, signal} from '@angular/core'; -import {RunGroup, RunInfo} from '../../../../runner/shared-interfaces'; +import {RunGroup, RunInfoFromReportServer} from '../../../../runner/shared-interfaces'; import {isPlatformBrowser} from '@angular/common'; @Injectable({providedIn: 'root'}) export class ReportsFetcher { private readonly platformId = inject(PLATFORM_ID); private readonly pendingFetches = signal(0); - private readonly runCache = new Map(); + private readonly runCache = new Map(); private readonly groupsResource = resource({ loader: async () => { if (!isPlatformBrowser(this.platformId)) { @@ -36,7 +36,7 @@ export class ReportsFetcher { readonly isLoadingSingleReport = computed(() => this.pendingFetches() > 0); readonly isLoadingReportsList = computed(() => this.groupsResource.isLoading()); - async getCombinedReport(groupId: string): Promise { + async getCombinedReport(groupId: string): Promise { if (!this.runCache.has(groupId)) { this.pendingFetches.update(current => current + 1); @@ -47,7 +47,7 @@ export class ReportsFetcher { throw new Error(`Response status: ${response.status}`); } - const allRuns = (await response.json()) as RunInfo[]; + const allRuns = (await response.json()) as RunInfoFromReportServer[]; if (!Array.isArray(allRuns) || allRuns.length === 0) { throw new Error(`Could not find report with id: ${groupId}`); @@ -59,7 +59,7 @@ export class ReportsFetcher { group: firstRun.group, details: firstRun.details, results: allRuns.flatMap(run => run.results), - } satisfies RunInfo; + } satisfies RunInfoFromReportServer; this.runCache.set(groupId, combined); } finally { diff --git a/report-app/src/app/shared/ai-assistant/ai-assistant.html b/report-app/src/app/shared/ai-assistant/ai-assistant.html index 58f701c..570ee7a 100644 --- a/report-app/src/app/shared/ai-assistant/ai-assistant.html +++ b/report-app/src/app/shared/ai-assistant/ai-assistant.html @@ -6,13 +6,6 @@

AI Assistant

-
- -
+
+ + + @if (isContextExpanded()) { +
+
+ + +
+
+ + +
+
+ + +
+
+ } +