From de641209ea01b5cee74e02cfc653794561cfddc1 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 22 Mar 2026 22:33:22 +0800 Subject: [PATCH] feat(q-dev): add Steering & Spec Mode Adoption dashboard Track Kiro steering rules and spec mode adoption: - User/request adoption rate stats - Weekly adoption rate trend - Steering impact on prompt/response length - Per-user feature adoption table --- .../dashboards/SteeringAdoptionTracker.json | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 grafana/dashboards/SteeringAdoptionTracker.json diff --git a/grafana/dashboards/SteeringAdoptionTracker.json b/grafana/dashboards/SteeringAdoptionTracker.json new file mode 100644 index 00000000000..4a840893827 --- /dev/null +++ b/grafana/dashboards/SteeringAdoptionTracker.json @@ -0,0 +1,212 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Steering", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "purple" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Spec Mode", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Requests with Steering", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Weekly trend of steering and spec mode adoption rate", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "id": 4, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Steering Rate',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Spec Mode Rate'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Steering & Spec Mode Adoption Rate", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Compare prompt and response quality between steering and non-steering sessions", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 5, + "options": { + "barRadius": 0.1, "barWidth": 0.5, + "orientation": "horizontal", "showValue": "auto", "stacking": "none", + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'With Steering' AS 'Mode', ROUND(AVG(prompt_length)) AS 'Avg Prompt', ROUND(AVG(response_length)) AS 'Avg Response'\nFROM _tool_q_dev_chat_log WHERE has_steering = 1 AND $__timeFilter(timestamp)\nUNION ALL\nSELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length))\nFROM _tool_q_dev_chat_log WHERE has_steering = 0 AND $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Impact: Prompt & Response Length", + "type": "bargauge" + }, + { + "datasource": "mysql", + "description": "Per-user steering and spec mode usage", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 6, + "options": { "cellHeight": "sm", "showHeader": true }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) AS 'User',\n COUNT(*) AS 'Total Chats',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'Steering %'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Per-User Feature Adoption", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "kiro", "steering", "adoption"], + "templating": { "list": [] }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Steering & Spec Mode Adoption", + "uid": "kiro_steering_adoption", + "version": 1 +}