From f0c57d02d7e09c61cbd7f351c464fecee18f3bc2 Mon Sep 17 00:00:00 2001
From: Paul Gschwendtner <paulgschwendtner@gmail.com>
Date: Thu, 2 Oct 2025 12:10:22 +0000
Subject: [PATCH] feat: allow for AI assist to reason about our score system

This allows users to ask questions like: "What are the failures for apps
in the "Poor" category".
---
 .../app/shared/ai-assistant/ai-assistant.html  |  4 ++--
 .../app/shared/ai-assistant/ai-assistant.scss  | 16 ++++------------
 .../app/shared/ai-assistant/ai-assistant.ts    |  5 ++++-
 runner/orchestration/generate.ts               |  1 -
 runner/ratings/stats.ts                        |  2 +-
 runner/reporting/report-ai-chat.ts             | 18 ++++++++++++++++--
 6 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/report-app/src/app/shared/ai-assistant/ai-assistant.html b/report-app/src/app/shared/ai-assistant/ai-assistant.html
index 2f2ae63..2f73fc3 100644
--- a/report-app/src/app/shared/ai-assistant/ai-assistant.html
+++ b/report-app/src/app/shared/ai-assistant/ai-assistant.html
@@ -4,6 +4,8 @@
       <div class="title-row">
         <h2>AI Assistant</h2>
       </div>
+    </div>
+    <div class="header-actions">
       <div class="model-selector-container">
         <select [(ngModel)]="selectedModel">
           @for (model of models(); track model) {
@@ -11,8 +13,6 @@ <h2>AI Assistant</h2>
           }
         </select>
       </div>
-    </div>
-    <div class="header-actions">
       <button class="expand-button" (click)="toggleExpanded()">
         <span class="material-symbols-outlined">
           @if (isExpanded()) {
diff --git a/report-app/src/app/shared/ai-assistant/ai-assistant.scss b/report-app/src/app/shared/ai-assistant/ai-assistant.scss
index 8cc49ba..2ce7372 100644
--- a/report-app/src/app/shared/ai-assistant/ai-assistant.scss
+++ b/report-app/src/app/shared/ai-assistant/ai-assistant.scss
@@ -61,18 +61,10 @@
     color: var(--text-secondary);
 
     select {
-      background-color: var(--background-color);
-      color: var(--text-primary);
-      border: 1px solid var(--border-color);
-      border-radius: 6px;
-      font-size: 13px;
-      padding: 2px 6px;
-      cursor: pointer;
-
-      &:focus {
-        outline: none;
-        border-color: var(--accent-blue);
-      }
+      padding-left: 6px;
+      width: 100%;
+      font-size: 0.8rem;
+      height: 1.5rem;
     }
   }
 
diff --git a/report-app/src/app/shared/ai-assistant/ai-assistant.ts b/report-app/src/app/shared/ai-assistant/ai-assistant.ts
index 6656868..08b9baf 100644
--- a/report-app/src/app/shared/ai-assistant/ai-assistant.ts
+++ b/report-app/src/app/shared/ai-assistant/ai-assistant.ts
@@ -32,7 +32,10 @@ export class AiAssistant {
 
   protected readonly aiConfigState = httpResource<AIConfigState>(() => '/api/ai-config-state');
   protected readonly models = computed(() => this.aiConfigState.value()?.configuredModels ?? []);
-  protected selectedModel = linkedSignal(() => this.models()[0]);
+  protected selectedModel = linkedSignal(
+    // By default, select Gemini 2.5 Flash if available (it's fast enough for chat + can reason well).
+    () => this.models().find(m => m === 'gemini-2.5-flash') ?? this.models()[0],
+  );
 
   protected toggleExpanded(): void {
     this.isExpanded.set(!this.isExpanded());
diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts
index 31b2715..95b12c4 100644
--- a/runner/orchestration/generate.ts
+++ b/runner/orchestration/generate.ts
@@ -17,7 +17,6 @@ import {
 } from '../configuration/constants.js';
 import {Environment} from '../configuration/environment.js';
 import {rateGeneratedCode} from '../ratings/rate-code.js';
-import {chatWithReportAI} from '../reporting/report-ai-chat.js';
 import {redX} from '../reporting/format.js';
 import {
   AssessmentResult,
diff --git a/runner/ratings/stats.ts b/runner/ratings/stats.ts
index d278d87..7d94753 100644
--- a/runner/ratings/stats.ts
+++ b/runner/ratings/stats.ts
@@ -8,7 +8,7 @@ import {
 } from '../shared-interfaces.js';
 
 /** Possible buckets that scores can be categorized into. */
-const BUCKET_CONFIG = [
+export const BUCKET_CONFIG = [
   {name: 'Excellent', min: 98, max: 100, id: 'excellent'},
   {name: 'Great', min: 85, max: 97, id: 'great'},
   {name: 'Good', min: 71, max: 84, id: 'good'},
diff --git a/runner/reporting/report-ai-chat.ts b/runner/reporting/report-ai-chat.ts
index b99ec08..7f0ae71 100644
--- a/runner/reporting/report-ai-chat.ts
+++ b/runner/reporting/report-ai-chat.ts
@@ -7,6 +7,8 @@ import {
   IndividualAssessmentState,
 } from '../shared-interfaces.js';
 import {BuildResultStatus} from '../workers/builder/builder-types.js';
+import {BUCKET_CONFIG} from '../ratings/stats.js';
+import {POINTS_FOR_CATEGORIES} from '../ratings/rating-types.js';
 
 export const reportLlmEvalsToolContext = `## What is a report?
 A report consists of many apps that were LLM generated. You will have information
@@ -15,6 +17,17 @@ about checks that failed for this LLM generated app.
 Note that there may be multiple attempts for an app. E.g. an initial build may fail and
 another attempt might have repaired the build failure. The last attempt reflects the final
 state of the app. E.g. whether it does build, or if there are runtime errors.
+
+## Scoring mechanism
+Apps are rated based on their scores in the following buckets:
+${BUCKET_CONFIG.map(b => `* ${b.name}: ${b.min}-${b.max}`).join('\n')}
+
+The overall score of an app is determined based on score reductions.
+There are three pillars: ${Object.keys(POINTS_FOR_CATEGORIES).join(', ')}
+Pillars are a split up of a 100% perfect score, allowing for individual ratings
+to be less impactful than others. The pillars are distributed as follows:
+${Object.entries(POINTS_FOR_CATEGORIES).map(e => `* ${e[0]}: ${e[1]} points.`)}
+Within pillars, the available score can be reduced by individual ratings.
 `;
 
 const defaultAiChatPrompt = `Strictly follow the instructions here.
@@ -86,7 +99,7 @@ export function serializeReportForPrompt(assessments: AssessmentResult[]): strin
         `
 Name: ${app.promptDef.name}
 Score: ${app.score.totalPoints}/${app.score.maxOverallPoints}
-Failed checks: ${JSON.stringify(
+Failed checks/ratings: ${JSON.stringify(
           app.score.categories
             .flatMap(category => category.assessments)
             .filter(
@@ -95,7 +108,8 @@ Failed checks: ${JSON.stringify(
             )
             .map(c => ({
               description: c.description,
-              points: `${(c.successPercentage * 100).toFixed(2)}/100`,
+              category: c.category,
+              scoreReduction: c.scoreReduction,
               message: c.message,
             })),
           null,