From 048ce43c3aa3d7ed0b8fcab92a1355e0c75d0c76 Mon Sep 17 00:00:00 2001 From: warren Date: Sat, 14 Mar 2026 23:46:09 +0800 Subject: [PATCH 01/21] feat(q-dev): add logging data ingestion and enrich Kiro dashboards Add support for ingesting S3 logging data (GenerateAssistantResponse and GenerateCompletions events) into new database tables, and enrich all three Kiro Grafana dashboards with additional metrics. Changes: - New models: QDevChatLog and QDevCompletionLog for logging event data - New extractor: s3_logging_extractor.go parses JSON.gz logging files - Updated S3 collector to also handle .json.gz files - Added logging S3 prefixes (GenerateAssistantResponse, GenerateCompletions) - New dashboard: "Kiro AI Activity Insights" with 10 panels including model usage distribution, active hours, conversation depth, feature adoption (Steering/Spec), file type usage, and prompt/response trends - Enriched "Kiro Code Metrics Dashboard" with DocGeneration, TestGeneration, and Dev (Agentic) metric panels - Fixed "Kiro Usage Dashboard" per-user table to sort by user_id - Migration script for new tables --- backend/plugins/q_dev/impl/impl.go | 5 + backend/plugins/q_dev/impl/impl_test.go | 4 +- backend/plugins/q_dev/models/chat_log.go | 51 ++ .../plugins/q_dev/models/completion_log.go | 43 + .../20260314_add_logging_tables.go | 43 + .../migrationscripts/archived/chat_log.go | 50 ++ .../archived/completion_log.go | 42 + .../q_dev/models/migrationscripts/register.go | 1 + .../plugins/q_dev/tasks/s3_file_collector.go | 6 +- .../q_dev/tasks/s3_logging_extractor.go | 308 +++++++ grafana/dashboards/qdev_logging.json | 800 ++++++++++++++++++ grafana/dashboards/qdev_user_data.json | 389 ++++++++- grafana/dashboards/qdev_user_report.json | 2 +- 13 files changed, 1737 insertions(+), 7 deletions(-) create mode 100644 backend/plugins/q_dev/models/chat_log.go create mode 100644 backend/plugins/q_dev/models/completion_log.go create mode 100644 backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go create mode 100644 backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go create mode 100644 backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go create mode 100644 backend/plugins/q_dev/tasks/s3_logging_extractor.go create mode 100644 grafana/dashboards/qdev_logging.json diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index e38fe7ad73c..a0435e58bd9 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -58,6 +58,8 @@ func (p QDev) GetTablesInfo() []dal.Tabler { &models.QDevS3FileMeta{}, &models.QDevS3Slice{}, &models.QDevUserReport{}, + &models.QDevChatLog{}, + &models.QDevCompletionLog{}, } } @@ -85,6 +87,7 @@ func (p QDev) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ tasks.CollectQDevS3FilesMeta, tasks.ExtractQDevS3DataMeta, + tasks.ExtractQDevLoggingDataMeta, } } @@ -131,6 +134,8 @@ func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int s3Prefixes = []string{ fmt.Sprintf("%s/by_user_analytic/%s/%s", base, region, timePart), fmt.Sprintf("%s/user_report/%s/%s", base, region, timePart), + fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", base, region, timePart), + fmt.Sprintf("%s/GenerateCompletions/%s/%s", base, region, timePart), } } else { // Legacy scope: use S3Prefix directly diff --git a/backend/plugins/q_dev/impl/impl_test.go b/backend/plugins/q_dev/impl/impl_test.go index e61b5325162..7153617ab9a 100644 --- a/backend/plugins/q_dev/impl/impl_test.go +++ b/backend/plugins/q_dev/impl/impl_test.go @@ -34,11 +34,11 @@ func TestQDev_BasicPluginMethods(t *testing.T) { // Test table info tables := plugin.GetTablesInfo() - assert.Len(t, tables, 5) + assert.Len(t, tables, 7) // Test subtask metas subtasks := plugin.SubTaskMetas() - assert.Len(t, subtasks, 2) + assert.Len(t, subtasks, 3) // Test API resources apiResources := plugin.ApiResources() diff --git a/backend/plugins/q_dev/models/chat_log.go b/backend/plugins/q_dev/models/chat_log.go new file mode 100644 index 00000000000..265722af5ff --- /dev/null +++ b/backend/plugins/q_dev/models/chat_log.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevChatLog stores parsed data from GenerateAssistantResponse logging events +type QDevChatLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/completion_log.go b/backend/plugins/q_dev/models/completion_log.go new file mode 100644 index 00000000000..0d0e0404ce8 --- /dev/null +++ b/backend/plugins/q_dev/models/completion_log.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevCompletionLog stores parsed data from GenerateCompletions logging events +type QDevCompletionLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go new file mode 100644 index 00000000000..cbd5943ecd2 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" +) + +type addLoggingTables struct{} + +func (*addLoggingTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevChatLog{}, + &archived.QDevCompletionLog{}, + ) +} + +func (*addLoggingTables) Version() uint64 { + return 20260314000001 +} + +func (*addLoggingTables) Name() string { + return "Add chat_log and completion_log tables for Kiro logging data" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go new file mode 100644 index 00000000000..ae766e301cb --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -0,0 +1,50 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevChatLog struct { + archived.Model + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go new file mode 100644 index 00000000000..1e915ff9bbd --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevCompletionLog struct { + archived.Model + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 9c68ae8f815..5480d5eaf29 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go @@ -35,5 +35,6 @@ func All() []plugin.MigrationScript { new(addAccountIdToS3Slice), new(fixDedupUserTables), new(resetS3FileMetaProcessed), + new(addLoggingTables), } } diff --git a/backend/plugins/q_dev/tasks/s3_file_collector.go b/backend/plugins/q_dev/tasks/s3_file_collector.go index 9d40919ae76..1ab4f8f0aa7 100644 --- a/backend/plugins/q_dev/tasks/s3_file_collector.go +++ b/backend/plugins/q_dev/tasks/s3_file_collector.go @@ -59,9 +59,9 @@ func CollectQDevS3Files(taskCtx plugin.SubTaskContext) errors.Error { } for _, object := range result.Contents { - // Only process CSV files - if !strings.HasSuffix(*object.Key, ".csv") { - taskCtx.GetLogger().Debug("Skipping non-CSV file: %s", *object.Key) + // Only process CSV and JSON.gz files + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + taskCtx.GetLogger().Debug("Skipping unsupported file: %s", *object.Key) continue } diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go new file mode 100644 index 00000000000..f25cc965903 --- /dev/null +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -0,0 +1,308 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/q_dev/models" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +var _ plugin.SubTaskEntryPoint = ExtractQDevLoggingData + +// ExtractQDevLoggingData extracts logging data from S3 JSON.gz files +func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*QDevTaskData) + db := taskCtx.GetDal() + + cursor, err := db.Cursor( + dal.From(&models.QDevS3FileMeta{}), + dal.Where("connection_id = ? AND processed = ? AND file_name LIKE ?", + data.Options.ConnectionId, false, "%.json.gz"), + ) + if err != nil { + return errors.Default.Wrap(err, "failed to get logging file metadata cursor") + } + defer cursor.Close() + + taskCtx.SetProgress(0, -1) + + for cursor.Next() { + fileMeta := &models.QDevS3FileMeta{} + err = db.Fetch(cursor, fileMeta) + if err != nil { + return errors.Default.Wrap(err, "failed to fetch file metadata") + } + + getInput := &s3.GetObjectInput{ + Bucket: aws.String(data.S3Client.Bucket), + Key: aws.String(fileMeta.S3Path), + } + + getResult, err := data.S3Client.S3.GetObject(getInput) + if err != nil { + return errors.Convert(err) + } + + tx := db.Begin() + processErr := processLoggingData(taskCtx, tx, getResult.Body, fileMeta) + if processErr != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + } + return errors.Default.Wrap(processErr, fmt.Sprintf("failed to process logging file %s", fileMeta.FileName)) + } + + fileMeta.Processed = true + now := time.Now() + fileMeta.ProcessedTime = &now + err = tx.Update(fileMeta) + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + } + return errors.Default.Wrap(err, "failed to update file metadata") + } + + err = tx.Commit() + if err != nil { + return errors.Default.Wrap(err, "failed to commit transaction") + } + + taskCtx.IncProgress(1) + } + + return nil +} + +// JSON structures for logging data + +type loggingFile struct { + Records []json.RawMessage `json:"records"` +} + +type chatLogRecord struct { + Request *chatLogRequest `json:"generateAssistantResponseEventRequest"` + Response *chatLogResponse `json:"generateAssistantResponseEventResponse"` +} + +type chatLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + ChatTriggerType string `json:"chatTriggerType"` + CustomizationArn *string `json:"customizationArn"` + ModelID string `json:"modelId"` + Prompt string `json:"prompt"` +} + +type chatLogResponse struct { + RequestID string `json:"requestId"` + AssistantResponse string `json:"assistantResponse"` + MessageMetadata struct { + ConversationID *string `json:"conversationId"` + UtteranceID *string `json:"utteranceId"` + } `json:"messageMetadata"` +} + +type completionLogRecord struct { + Request *completionLogRequest `json:"generateCompletionsEventRequest"` + Response *completionLogResponse `json:"generateCompletionsEventResponse"` +} + +type completionLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + FileName string `json:"fileName"` + CustomizationArn *string `json:"customizationArn"` +} + +type completionLogResponse struct { + RequestID string `json:"requestId"` + Completions []json.RawMessage `json:"completions"` +} + +func processLoggingData(taskCtx plugin.SubTaskContext, db dal.Dal, reader io.ReadCloser, fileMeta *models.QDevS3FileMeta) errors.Error { + defer reader.Close() + + data := taskCtx.GetData().(*QDevTaskData) + + gzReader, err := gzip.NewReader(reader) + if err != nil { + return errors.Convert(err) + } + defer gzReader.Close() + + var logFile loggingFile + decoder := json.NewDecoder(gzReader) + if err := decoder.Decode(&logFile); err != nil { + return errors.Convert(err) + } + + isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse") + + for _, rawRecord := range logFile.Records { + if isChatLog { + if err := processChatRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { + return err + } + } else { + if err := processCompletionRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { + return err + } + } + } + + return nil +} + +func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { + var record chatLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return errors.Convert(err) + } + + if record.Request == nil || record.Response == nil { + return nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + chatLog := &models.QDevChatLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: record.Request.UserID, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + Timestamp: ts, + ChatTriggerType: record.Request.ChatTriggerType, + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + ModelId: record.Request.ModelID, + PromptLength: len(record.Request.Prompt), + ResponseLength: len(record.Response.AssistantResponse), + } + + // Parse structured info from prompt + prompt := record.Request.Prompt + chatLog.OpenFileCount = countOpenFiles(prompt) + chatLog.ActiveFileName, chatLog.ActiveFileExtension = parseActiveFile(prompt) + chatLog.HasSteering = strings.Contains(prompt, ".kiro/steering") + chatLog.IsSpecMode = strings.Contains(prompt, "implicit-rules") + + if record.Response.MessageMetadata.ConversationID != nil { + chatLog.ConversationId = *record.Response.MessageMetadata.ConversationID + } + if record.Response.MessageMetadata.UtteranceID != nil { + chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID + } + + return errors.Default.Wrap(db.CreateOrUpdate(chatLog), "failed to save chat log") +} + +// countOpenFiles counts tags within block +func countOpenFiles(prompt string) int { + start := strings.Index(prompt, "") + if start == -1 { + return 0 + } + end := strings.Index(prompt, "") + if end == -1 { + return 0 + } + block := prompt[start:end] + return strings.Count(block, "") + if start == -1 { + return "", "" + } + end := strings.Index(prompt[start:], "") + if end == -1 { + return "", "" + } + block := prompt[start : start+end] + // Find + nameStart := strings.Index(block, "name=\"") + if nameStart == -1 { + return "", "" + } + nameStart += len("name=\"") + nameEnd := strings.Index(block[nameStart:], "\"") + if nameEnd == -1 { + return "", "" + } + fileName := block[nameStart : nameStart+nameEnd] + ext := filepath.Ext(fileName) + return fileName, ext +} + +func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { + var record completionLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return errors.Convert(err) + } + + if record.Request == nil || record.Response == nil { + return nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + completionLog := &models.QDevCompletionLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: record.Request.UserID, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + Timestamp: ts, + FileName: record.Request.FileName, + FileExtension: filepath.Ext(record.Request.FileName), + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + CompletionsCount: len(record.Response.Completions), + } + + return errors.Default.Wrap(db.CreateOrUpdate(completionLog), "failed to save completion log") +} + +var ExtractQDevLoggingDataMeta = plugin.SubTaskMeta{ + Name: "extractQDevLoggingData", + EntryPoint: ExtractQDevLoggingData, + EnabledByDefault: true, + Description: "Extract logging data from S3 JSON.gz files (chat and completion events)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta}, +} diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json new file mode 100644 index 00000000000..b6d0627a56e --- /dev/null +++ b/grafana/dashboards/qdev_logging.json @@ -0,0 +1,800 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "mysql", + "description": "Overview of logging event metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Users'", + "refId": "A" + } + ], + "title": "Logging Overview", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Hourly distribution of AI usage activity (chat + completions)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hour of Day", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n hour_of_day as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", + "refId": "A" + } + ], + "title": "Active Hours Distribution", + "type": "barchart" + }, + { + "datasource": "mysql", + "description": "Distribution of model usage across chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 3, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN model_id = '' OR model_id IS NULL THEN '(unknown)'\n ELSE model_id\n END as 'Model',\n COUNT(*) as 'Requests'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY model_id\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Model Usage Distribution", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Top file extensions used with inline completions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "File Type Usage (Completions)", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Average number of chat events per conversation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n COUNT(*) / NULLIF(COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END), 0) as 'Avg Turns per Conversation',\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as 'Unique Conversations',\n COUNT(*) as 'Total Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Conversation Depth Analysis", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily chat and completion events over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", + "refId": "A" + } + ], + "title": "Daily Event Trends", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Per-user logging activity summary", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", + "refId": "A" + } + ], + "title": "Per-User Activity", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Kiro Feature Adoption", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Top file extensions active during chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 9, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)'\n ELSE active_file_extension\n END as 'File Type',\n COUNT(*) as 'Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "Active File Types in Chat", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Average and maximum prompt/response lengths over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Prompt & Response Length Trends", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "logging", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro AI Activity Insights", + "uid": "qdev_logging", + "version": 1 +} diff --git a/grafana/dashboards/qdev_user_data.json b/grafana/dashboards/qdev_user_data.json index d80d57bab21..e66901a5dbf 100644 --- a/grafana/dashboards/qdev_user_data.json +++ b/grafana/dashboards/qdev_user_data.json @@ -730,7 +730,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name\nORDER BY SUM(inline_ai_code_lines) DESC", + "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name\nORDER BY SUM(inline_ai_code_lines) DESC", "refId": "A", "select": [ [ @@ -771,6 +771,393 @@ ], "title": "User Interactions", "type": "table" + }, + { + "datasource": "mysql", + "description": "Daily doc generation events and accepted/rejected lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Generation Events',\n SUM(doc_generation_accepted_line_additions) as 'Accepted Line Additions',\n SUM(doc_generation_accepted_line_updates) as 'Accepted Line Updates',\n SUM(doc_generation_rejected_line_additions) as 'Rejected Line Additions',\n SUM(doc_generation_rejected_line_updates) as 'Rejected Line Updates'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Doc Generation Metrics", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily test generation events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Generation Events',\n SUM(test_generation_accepted_tests) as 'Accepted Tests',\n SUM(test_generation_generated_tests) as 'Generated Tests',\n SUM(test_generation_accepted_lines) as 'Accepted Lines',\n SUM(test_generation_generated_lines) as 'Generated Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Test Generation Metrics", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily agentic dev events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Dev Generation Events',\n SUM(dev_acceptance_event_count) as 'Dev Acceptance Events',\n SUM(dev_generated_lines) as 'Dev Generated Lines',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Dev (Agentic) Metrics", + "type": "timeseries" } ], "preload": false, diff --git a/grafana/dashboards/qdev_user_report.json b/grafana/dashboards/qdev_user_report.json index e1a27bc539b..68f4b8650ba 100644 --- a/grafana/dashboards/qdev_user_report.json +++ b/grafana/dashboards/qdev_user_report.json @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name, subscription_tier, client_type\nORDER BY SUM(credits_used) DESC", + "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name, subscription_tier, client_type\nORDER BY user_id", "refId": "A" } ], From ac7622d0b5037d8d48b256ddca84bf9f7b6d26a2 Mon Sep 17 00:00:00 2001 From: warren Date: Sat, 14 Mar 2026 23:55:40 +0800 Subject: [PATCH 02/21] fix(q-dev): use separate base path for logging S3 prefixes Logging data lives under a different S3 prefix ("logging/") than user report data ("user-report/"). Add LoggingBasePath option (defaults to "logging") so logging prefixes are constructed correctly. --- backend/plugins/q_dev/impl/impl.go | 9 +++++++-- backend/plugins/q_dev/tasks/task_data.go | 15 ++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index a0435e58bd9..46516b83c51 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -131,11 +131,16 @@ func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int timePart = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) } base := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", op.BasePath, op.AccountId) + loggingBasePath := op.LoggingBasePath + if loggingBasePath == "" { + loggingBasePath = "logging" + } + loggingBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", loggingBasePath, op.AccountId) s3Prefixes = []string{ fmt.Sprintf("%s/by_user_analytic/%s/%s", base, region, timePart), fmt.Sprintf("%s/user_report/%s/%s", base, region, timePart), - fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", base, region, timePart), - fmt.Sprintf("%s/GenerateCompletions/%s/%s", base, region, timePart), + fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", loggingBase, region, timePart), + fmt.Sprintf("%s/GenerateCompletions/%s/%s", loggingBase, region, timePart), } } else { // Legacy scope: use S3Prefix directly diff --git a/backend/plugins/q_dev/tasks/task_data.go b/backend/plugins/q_dev/tasks/task_data.go index 3fd3c65848d..6436a7f0580 100644 --- a/backend/plugins/q_dev/tasks/task_data.go +++ b/backend/plugins/q_dev/tasks/task_data.go @@ -26,13 +26,14 @@ type QDevApiParams struct { } type QDevOptions struct { - ConnectionId uint64 `json:"connectionId"` - S3Prefix string `json:"s3Prefix"` - ScopeId string `json:"scopeId"` - AccountId string `json:"accountId"` - BasePath string `json:"basePath"` - Year int `json:"year"` - Month *int `json:"month"` + ConnectionId uint64 `json:"connectionId"` + S3Prefix string `json:"s3Prefix"` + ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + BasePath string `json:"basePath"` + LoggingBasePath string `json:"loggingBasePath"` + Year int `json:"year"` + Month *int `json:"month"` } type QDevTaskData struct { From 04b8ea593be6da1f4eefef28224340019998388b Mon Sep 17 00:00:00 2001 From: warren Date: Sat, 14 Mar 2026 23:59:54 +0800 Subject: [PATCH 03/21] fix(q-dev): auto-scan logging path without extra config Kiro exports to two well-known S3 prefixes in the same bucket: - user-report/AWSLogs/{accountId}/KiroLogs/ (CSV reports) - logging/AWSLogs/{accountId}/KiroLogs/ (interaction logs) When AccountId is set, automatically scan both paths. The "logging" prefix is hardcoded since it's a standard Kiro export convention. No additional configuration needed. --- backend/plugins/q_dev/impl/impl.go | 22 +++++++++++++--------- backend/plugins/q_dev/tasks/task_data.go | 15 +++++++-------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 46516b83c51..3c6e1ed1644 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -130,17 +130,21 @@ func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int if op.Month != nil { timePart = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) } - base := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", op.BasePath, op.AccountId) - loggingBasePath := op.LoggingBasePath - if loggingBasePath == "" { - loggingBasePath = "logging" + // Kiro exports data to two well-known S3 prefixes: + // {basePath}/AWSLogs/{accountId}/KiroLogs/ — user report CSVs + // logging/AWSLogs/{accountId}/KiroLogs/ — interaction logs (JSON.gz) + // When basePath is empty, default to "user-report" for CSV data. + reportBase := op.BasePath + if reportBase == "" { + reportBase = "user-report" } - loggingBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", loggingBasePath, op.AccountId) + csvBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", reportBase, op.AccountId) + logBase := fmt.Sprintf("logging/AWSLogs/%s/KiroLogs", op.AccountId) s3Prefixes = []string{ - fmt.Sprintf("%s/by_user_analytic/%s/%s", base, region, timePart), - fmt.Sprintf("%s/user_report/%s/%s", base, region, timePart), - fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", loggingBase, region, timePart), - fmt.Sprintf("%s/GenerateCompletions/%s/%s", loggingBase, region, timePart), + fmt.Sprintf("%s/by_user_analytic/%s/%s", csvBase, region, timePart), + fmt.Sprintf("%s/user_report/%s/%s", csvBase, region, timePart), + fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", logBase, region, timePart), + fmt.Sprintf("%s/GenerateCompletions/%s/%s", logBase, region, timePart), } } else { // Legacy scope: use S3Prefix directly diff --git a/backend/plugins/q_dev/tasks/task_data.go b/backend/plugins/q_dev/tasks/task_data.go index 6436a7f0580..3fd3c65848d 100644 --- a/backend/plugins/q_dev/tasks/task_data.go +++ b/backend/plugins/q_dev/tasks/task_data.go @@ -26,14 +26,13 @@ type QDevApiParams struct { } type QDevOptions struct { - ConnectionId uint64 `json:"connectionId"` - S3Prefix string `json:"s3Prefix"` - ScopeId string `json:"scopeId"` - AccountId string `json:"accountId"` - BasePath string `json:"basePath"` - LoggingBasePath string `json:"loggingBasePath"` - Year int `json:"year"` - Month *int `json:"month"` + ConnectionId uint64 `json:"connectionId"` + S3Prefix string `json:"s3Prefix"` + ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + BasePath string `json:"basePath"` + Year int `json:"year"` + Month *int `json:"month"` } type QDevTaskData struct { From 8796f0f0e30eff67d906aee2b41c0e9cbf2b0df1 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 00:04:30 +0800 Subject: [PATCH 04/21] fix(q-dev): update scope tooltip to mention logging data scanning --- config-ui/src/plugins/register/q-dev/data-scope.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-ui/src/plugins/register/q-dev/data-scope.tsx b/config-ui/src/plugins/register/q-dev/data-scope.tsx index c5eff68db6a..657d0bdb890 100644 --- a/config-ui/src/plugins/register/q-dev/data-scope.tsx +++ b/config-ui/src/plugins/register/q-dev/data-scope.tsx @@ -326,7 +326,7 @@ export const QDevDataScope = ({ const timePart = meta.month ? `${meta.year}/${ensureLeadingZero(meta.month)}` : `${meta.year}`; return ( {meta.basePath}/…/{meta.accountId}/…/{timePart} From 68b7656bbe604c827b56bb22b57a3db30dc64c04 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 00:11:56 +0800 Subject: [PATCH 05/21] fix(q-dev): fix scope ID routing and CSV/JSON file separation Three fixes: 1. Use *scopeId (catch-all) route pattern instead of :scopeId so scope IDs containing "/" (e.g. "034362076319/2026") work in URL paths 2. CSV extractor now filters for .csv files only, preventing it from trying to parse .json.gz logging files as CSV 3. Frontend scope API calls now encodeURIComponent(scopeId) for safe URL encoding --- backend/plugins/q_dev/impl/impl.go | 4 ++-- backend/plugins/q_dev/tasks/s3_data_extractor.go | 5 +++-- config-ui/src/api/scope/index.ts | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 3c6e1ed1644..44bbd85baa5 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -188,12 +188,12 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "GET": api.GetScopeList, "PUT": api.PutScopes, }, - "connections/:connectionId/scopes/:scopeId": { + "connections/:connectionId/scopes/*scopeId": { "GET": api.GetScope, "PATCH": api.PatchScope, "DELETE": api.DeleteScope, }, - "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "connections/:connectionId/scopes/*scopeId/latest-sync-state": { "GET": api.GetScopeLatestSyncState, }, } diff --git a/backend/plugins/q_dev/tasks/s3_data_extractor.go b/backend/plugins/q_dev/tasks/s3_data_extractor.go index 1cf2a9f2ef5..7b586ce9e65 100644 --- a/backend/plugins/q_dev/tasks/s3_data_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_data_extractor.go @@ -40,10 +40,11 @@ func ExtractQDevS3Data(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*QDevTaskData) db := taskCtx.GetDal() - // 查询未处理的文件元数据 + // 查询未处理的CSV文件元数据（排除.json.gz日志文件） cursor, err := db.Cursor( dal.From(&models.QDevS3FileMeta{}), - dal.Where("connection_id = ? AND processed = ?", data.Options.ConnectionId, false), + dal.Where("connection_id = ? AND processed = ? AND file_name LIKE ?", + data.Options.ConnectionId, false, "%.csv"), ) if err != nil { return errors.Default.Wrap(err, "failed to get file metadata cursor") diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts index 7007314f694..39cfd818cfc 100644 --- a/config-ui/src/api/scope/index.ts +++ b/config-ui/src/api/scope/index.ts @@ -38,17 +38,17 @@ export const list = ( }); export const get = (plugin: string, connectionId: ID, scopeId: ID, payload?: { blueprints: boolean }) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}`, { data: payload, }); export const remove = (plugin: string, connectionId: ID, scopeId: ID, onlyData: boolean) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}?delete_data_only=${onlyData}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}?delete_data_only=${onlyData}`, { method: 'delete', }); export const update = (plugin: string, connectionId: ID, scopeId: ID, payload: any) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}`, { method: 'patch', data: payload, }); From cadaf64f3bf0c70be69e30bf473dd997a1153556 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 00:15:08 +0800 Subject: [PATCH 06/21] fix(q-dev): resolve *scopeId route conflict with dispatcher pattern The catch-all *scopeId route conflicts with *scopeId/latest-sync-state. Follow Jenkins/Bitbucket pattern: use a single *scopeId route with a GetScopeDispatcher that checks for /latest-sync-state suffix and dispatches accordingly. All scope handlers now TrimLeft "/" from scopeId. --- backend/plugins/q_dev/api/s3_slice_api.go | 59 +++++++---------------- backend/plugins/q_dev/impl/impl.go | 5 +- 2 files changed, 19 insertions(+), 45 deletions(-) diff --git a/backend/plugins/q_dev/api/s3_slice_api.go b/backend/plugins/q_dev/api/s3_slice_api.go index 73708130294..fa8a865864f 100644 --- a/backend/plugins/q_dev/api/s3_slice_api.go +++ b/backend/plugins/q_dev/api/s3_slice_api.go @@ -18,6 +18,8 @@ limitations under the License. package api import ( + "strings" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -59,63 +61,38 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er return dsHelper.ScopeApi.GetPage(input) } +// GetScopeDispatcher routes GET requests for *scopeId catch-all. +// When scopeId ends with /latest-sync-state, dispatches to GetScopeLatestSyncState. +// Otherwise dispatches to GetScope. +func GetScopeDispatcher(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + scopeIdWithSuffix := strings.TrimLeft(input.Params["scopeId"], "/") + if strings.HasSuffix(scopeIdWithSuffix, "/latest-sync-state") { + input.Params["scopeId"] = strings.TrimSuffix(scopeIdWithSuffix, "/latest-sync-state") + return GetScopeLatestSyncState(input) + } + return GetScope(input) +} + // GetScope returns a single scope record -// @Summary get a Q Developer scope -// @Description get a Q Developer scope -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param blueprints query bool false "include blueprint references" -// @Success 200 {object} ScopeDetail -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [GET] func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.GetScopeDetail(input) } // PatchScope updates a scope record -// @Summary patch a Q Developer scope -// @Description patch a Q Developer scope -// @Tags plugins/q_dev -// @Accept application/json -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param scope body models.QDevS3Slice true "json" -// @Success 200 {object} models.QDevS3Slice -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [PATCH] func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.Patch(input) } // DeleteScope removes a scope and optionally associated data. -// @Summary delete a Q Developer scope -// @Description delete Q Developer scope data -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param delete_data_only query bool false "Only delete scope data" -// @Success 200 -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [DELETE] func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.Delete(input) } // GetScopeLatestSyncState returns scope sync state info -// @Summary latest sync state for a Q Developer scope -// @Description get latest sync state for a Q Developer scope -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Success 200 -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.GetScopeLatestSyncState(input) } diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 44bbd85baa5..f7f9eaf7f1a 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -189,13 +189,10 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "PUT": api.PutScopes, }, "connections/:connectionId/scopes/*scopeId": { - "GET": api.GetScope, + "GET": api.GetScopeDispatcher, "PATCH": api.PatchScope, "DELETE": api.DeleteScope, }, - "connections/:connectionId/scopes/*scopeId/latest-sync-state": { - "GET": api.GetScopeLatestSyncState, - }, } } From fcf8f9cb591c9e3cd6c786c4ff4354cdc1d8d658 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 00:28:15 +0800 Subject: [PATCH 07/21] fix(q-dev): use URL-safe scope ID format (underscore separator) Scope IDs like "034362076319/2026" break URL routing because "/" is a path separator. Change ID format to "034362076319_2026" (underscore) when AccountId is set. The Prefix field still uses "/" for S3 path matching. Revert to standard :scopeId routes since IDs are now safe. Note: existing scopes need to be recreated after this change. --- backend/plugins/q_dev/api/s3_slice_api.go | 18 ------------------ backend/plugins/q_dev/impl/impl.go | 7 +++++-- backend/plugins/q_dev/models/s3_slice.go | 11 ++++++++++- config-ui/src/api/scope/index.ts | 6 +++--- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/backend/plugins/q_dev/api/s3_slice_api.go b/backend/plugins/q_dev/api/s3_slice_api.go index fa8a865864f..158aa4ef785 100644 --- a/backend/plugins/q_dev/api/s3_slice_api.go +++ b/backend/plugins/q_dev/api/s3_slice_api.go @@ -18,8 +18,6 @@ limitations under the License. package api import ( - "strings" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -61,38 +59,22 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er return dsHelper.ScopeApi.GetPage(input) } -// GetScopeDispatcher routes GET requests for *scopeId catch-all. -// When scopeId ends with /latest-sync-state, dispatches to GetScopeLatestSyncState. -// Otherwise dispatches to GetScope. -func GetScopeDispatcher(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - scopeIdWithSuffix := strings.TrimLeft(input.Params["scopeId"], "/") - if strings.HasSuffix(scopeIdWithSuffix, "/latest-sync-state") { - input.Params["scopeId"] = strings.TrimSuffix(scopeIdWithSuffix, "/latest-sync-state") - return GetScopeLatestSyncState(input) - } - return GetScope(input) -} - // GetScope returns a single scope record func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.GetScopeDetail(input) } // PatchScope updates a scope record func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.Patch(input) } // DeleteScope removes a scope and optionally associated data. func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.Delete(input) } // GetScopeLatestSyncState returns scope sync state info func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") return dsHelper.ScopeApi.GetScopeLatestSyncState(input) } diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index f7f9eaf7f1a..3c6e1ed1644 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -188,11 +188,14 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "GET": api.GetScopeList, "PUT": api.PutScopes, }, - "connections/:connectionId/scopes/*scopeId": { - "GET": api.GetScopeDispatcher, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, "PATCH": api.PatchScope, "DELETE": api.DeleteScope, }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, } } diff --git a/backend/plugins/q_dev/models/s3_slice.go b/backend/plugins/q_dev/models/s3_slice.go index e918258a908..19ecd6920bd 100644 --- a/backend/plugins/q_dev/models/s3_slice.go +++ b/backend/plugins/q_dev/models/s3_slice.go @@ -99,7 +99,16 @@ func (s *QDevS3Slice) normalize(strict bool) error { } if s.Id == "" { - s.Id = s.Prefix + if s.AccountId != "" { + // Use URL-safe ID: account_year or account_year_month + if s.Month != nil { + s.Id = fmt.Sprintf("%s_%04d_%02d", s.AccountId, s.Year, *s.Month) + } else { + s.Id = fmt.Sprintf("%s_%04d", s.AccountId, s.Year) + } + } else { + s.Id = s.Prefix + } } if s.AccountId != "" { diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts index 39cfd818cfc..7007314f694 100644 --- a/config-ui/src/api/scope/index.ts +++ b/config-ui/src/api/scope/index.ts @@ -38,17 +38,17 @@ export const list = ( }); export const get = (plugin: string, connectionId: ID, scopeId: ID, payload?: { blueprints: boolean }) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { data: payload, }); export const remove = (plugin: string, connectionId: ID, scopeId: ID, onlyData: boolean) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}?delete_data_only=${onlyData}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}?delete_data_only=${onlyData}`, { method: 'delete', }); export const update = (plugin: string, connectionId: ID, scopeId: ID, payload: any) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${encodeURIComponent(scopeId)}`, { + request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { method: 'patch', data: payload, }); From 77e610a7abbed51087e7284f62a57f64281fa728 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:12:33 +0800 Subject: [PATCH 08/21] fix(q-dev): use NoPKModel instead of Model in archived logging models archived.Model only has ID+timestamps, missing RawDataOrigin fields (_raw_data_params etc.) that common.NoPKModel includes. This caused "Unknown column '_raw_data_params'" errors at runtime. --- .../plugins/q_dev/models/migrationscripts/archived/chat_log.go | 2 +- .../q_dev/models/migrationscripts/archived/completion_log.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go index ae766e301cb..29de8174354 100644 --- a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -24,7 +24,7 @@ import ( ) type QDevChatLog struct { - archived.Model + archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go index 1e915ff9bbd..035c13ef2e0 100644 --- a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go +++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go @@ -24,7 +24,7 @@ import ( ) type QDevCompletionLog struct { - archived.Model + archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` From 29fcee3a525e151f704c333711281ac46f723491 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:24:37 +0800 Subject: [PATCH 09/21] fix(q-dev): fix GROUP BY in per-user table to merge display_name variants Remove display_name from GROUP BY so same user_id with different display_name values gets merged. Use MAX(display_name) in SELECT. --- grafana/dashboards/qdev_user_report.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/dashboards/qdev_user_report.json b/grafana/dashboards/qdev_user_report.json index 68f4b8650ba..88a12b6146d 100644 --- a/grafana/dashboards/qdev_user_report.json +++ b/grafana/dashboards/qdev_user_report.json @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name, subscription_tier, client_type\nORDER BY user_id", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id", "refId": "A" } ], From 84556cbc7260c65d1f84a84a8591537bf6b58526 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:31:05 +0800 Subject: [PATCH 10/21] fix(q-dev): normalize logging user IDs to match CSV short UUID format Logging data uses "d-{directoryId}.{UUID}" format while CSV user-report uses plain "{UUID}". Strip the "d-xxx." prefix so the same user maps to one user_id across both data sources. --- .../q_dev/tasks/s3_logging_extractor.go | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go index f25cc965903..9a782fd8085 100644 --- a/backend/plugins/q_dev/tasks/s3_logging_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -196,12 +196,13 @@ func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMe ts = time.Now() } + userId := normalizeUserId(record.Request.UserID) chatLog := &models.QDevChatLog{ ConnectionId: fileMeta.ConnectionId, ScopeId: fileMeta.ScopeId, RequestId: record.Response.RequestID, - UserId: record.Request.UserID, - DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + UserId: userId, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), userId, identityClient), Timestamp: ts, ChatTriggerType: record.Request.ChatTriggerType, HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", @@ -282,12 +283,13 @@ func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json ts = time.Now() } + userId := normalizeUserId(record.Request.UserID) completionLog := &models.QDevCompletionLog{ ConnectionId: fileMeta.ConnectionId, ScopeId: fileMeta.ScopeId, RequestId: record.Response.RequestID, - UserId: record.Request.UserID, - DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + UserId: userId, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), userId, identityClient), Timestamp: ts, FileName: record.Request.FileName, FileExtension: filepath.Ext(record.Request.FileName), @@ -298,6 +300,16 @@ func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json return errors.Default.Wrap(db.CreateOrUpdate(completionLog), "failed to save completion log") } +// normalizeUserId strips the "d-{directoryId}." prefix from Identity Center user IDs +// so that logging user IDs match the short UUID format used in user-report CSVs. +// e.g. "d-9067deb161.6478a4a8-60a1-70d9-37bc-6aae85f6746a" → "6478a4a8-60a1-70d9-37bc-6aae85f6746a" +func normalizeUserId(userId string) string { + if idx := strings.LastIndex(userId, "."); idx != -1 && strings.HasPrefix(userId, "d-") { + return userId[idx+1:] + } + return userId +} + var ExtractQDevLoggingDataMeta = plugin.SubTaskMeta{ Name: "extractQDevLoggingData", EntryPoint: ExtractQDevLoggingData, From 5c0c520596400947dd6a18e4a6519e9473d2779c Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:36:00 +0800 Subject: [PATCH 11/21] fix(q-dev): normalize user IDs in CSV extractors and sort table DESC Apply normalizeUserId to both createUserReportData and createUserDataWithDisplayName so user_report CSV data also strips the "d-{directoryId}." prefix. Change per-user table sort to ORDER BY user_id DESC. --- backend/plugins/q_dev/tasks/s3_data_extractor.go | 9 +++++---- grafana/dashboards/qdev_user_report.json | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/plugins/q_dev/tasks/s3_data_extractor.go b/backend/plugins/q_dev/tasks/s3_data_extractor.go index 7b586ce9e65..da29bca07fe 100644 --- a/backend/plugins/q_dev/tasks/s3_data_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_data_extractor.go @@ -203,8 +203,8 @@ func createUserReportData(logger interface { } } - // UserId - report.UserId = getStringField(fieldMap, "UserId") + // UserId (normalize to strip "d-{directoryId}." prefix if present) + report.UserId = normalizeUserId(getStringField(fieldMap, "UserId")) if report.UserId == "" { return nil, errors.Default.New("UserId not found in CSV record") } @@ -304,11 +304,12 @@ func createUserDataWithDisplayName(logger interface { var err error var ok bool - // 设置UserId - userData.UserId, ok = fieldMap["UserId"] + // 设置UserId (normalize to strip "d-{directoryId}." prefix if present) + rawUserId, ok := fieldMap["UserId"] if !ok { return nil, errors.Default.New("UserId not found in CSV record") } + userData.UserId = normalizeUserId(rawUserId) // 设置DisplayName (new functionality) userData.DisplayName = resolveDisplayName(logger, userData.UserId, identityClient) diff --git a/grafana/dashboards/qdev_user_report.json b/grafana/dashboards/qdev_user_report.json index 88a12b6146d..850a20fa1f4 100644 --- a/grafana/dashboards/qdev_user_report.json +++ b/grafana/dashboards/qdev_user_report.json @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", "refId": "A" } ], From faefafd2be922c4270adfa3fb1a40ea4dfbde63e Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:45:13 +0800 Subject: [PATCH 12/21] style(q-dev): fix gofmt formatting in chat_log models --- backend/plugins/q_dev/models/chat_log.go | 12 ++++++------ .../models/migrationscripts/archived/chat_log.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/backend/plugins/q_dev/models/chat_log.go b/backend/plugins/q_dev/models/chat_log.go index 265722af5ff..06679c515b6 100644 --- a/backend/plugins/q_dev/models/chat_log.go +++ b/backend/plugins/q_dev/models/chat_log.go @@ -26,12 +26,12 @@ import ( // QDevChatLog stores parsed data from GenerateAssistantResponse logging events type QDevChatLog struct { common.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` - RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` - UserId string `gorm:"index;type:varchar(255)" json:"userId"` - DisplayName string `gorm:"type:varchar(255)" json:"displayName"` - Timestamp time.Time `gorm:"index" json:"timestamp"` + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` HasCustomization bool `json:"hasCustomization"` ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go index 29de8174354..8278f52ff1e 100644 --- a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -25,12 +25,12 @@ import ( type QDevChatLog struct { archived.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` - RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` - UserId string `gorm:"index;type:varchar(255)" json:"userId"` - DisplayName string `gorm:"type:varchar(255)" json:"displayName"` - Timestamp time.Time `gorm:"index" json:"timestamp"` + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` HasCustomization bool `json:"hasCustomization"` ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` From 1e53e7c4533bd7a159d97c608fc6715a3d9418ff Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:48:51 +0800 Subject: [PATCH 13/21] perf(q-dev): parallelize logging S3 downloads and batch DB writes Optimize logging extractor performance: - 10 goroutine workers for parallel S3 file downloads - Batch 50 files per DB transaction instead of 1-per-file - sync.Map cache for display name resolution (avoid repeated IAM calls) - Parse records in memory during download, write all at once This should improve throughput from ~1.5 files/sec to ~15+ files/sec for typical logging file sizes. --- .../q_dev/tasks/s3_logging_extractor.go | 273 ++++++++++++------ 1 file changed, 191 insertions(+), 82 deletions(-) diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go index 9a782fd8085..2b28edc0a50 100644 --- a/backend/plugins/q_dev/tasks/s3_logging_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -20,10 +20,9 @@ package tasks import ( "compress/gzip" "encoding/json" - "fmt" - "io" "path/filepath" "strings" + "sync" "time" "github.com/apache/incubator-devlake/core/dal" @@ -36,6 +35,20 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractQDevLoggingData +const ( + loggingBatchSize = 50 // number of files to process per DB transaction + s3DownloadWorkers = 10 // parallel S3 download goroutines + s3DownloadChanSize = 20 // buffered channel size for download results +) + +// downloadResult holds the parsed records from one S3 file +type downloadResult struct { + FileMeta *models.QDevS3FileMeta + ChatLogs []*models.QDevChatLog + CompLogs []*models.QDevCompletionLog + Err error +} + // ExtractQDevLoggingData extracts logging data from S3 JSON.gz files func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*QDevTaskData) @@ -51,56 +64,190 @@ func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { } defer cursor.Close() - taskCtx.SetProgress(0, -1) - + // Collect all file metas first + var fileMetas []*models.QDevS3FileMeta for cursor.Next() { - fileMeta := &models.QDevS3FileMeta{} - err = db.Fetch(cursor, fileMeta) - if err != nil { + fm := &models.QDevS3FileMeta{} + if err := db.Fetch(cursor, fm); err != nil { return errors.Default.Wrap(err, "failed to fetch file metadata") } + fileMetas = append(fileMetas, fm) + } - getInput := &s3.GetObjectInput{ - Bucket: aws.String(data.S3Client.Bucket), - Key: aws.String(fileMeta.S3Path), - } + if len(fileMetas) == 0 { + return nil + } + + taskCtx.SetProgress(0, len(fileMetas)) + taskCtx.GetLogger().Info("Processing %d logging files with %d workers", len(fileMetas), s3DownloadWorkers) - getResult, err := data.S3Client.S3.GetObject(getInput) - if err != nil { - return errors.Convert(err) + // Display name cache to avoid repeated IAM calls + displayNameCache := &sync.Map{} + + // Process in batches + for batchStart := 0; batchStart < len(fileMetas); batchStart += loggingBatchSize { + batchEnd := batchStart + loggingBatchSize + if batchEnd > len(fileMetas) { + batchEnd = len(fileMetas) } + batch := fileMetas[batchStart:batchEnd] - tx := db.Begin() - processErr := processLoggingData(taskCtx, tx, getResult.Body, fileMeta) - if processErr != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + // Parallel download and parse + results := parallelDownloadAndParse(taskCtx, data, batch, displayNameCache) + + // Check for download errors + for _, r := range results { + if r.Err != nil { + return errors.Default.Wrap(errors.Convert(r.Err), + "failed to download/parse "+r.FileMeta.FileName) } - return errors.Default.Wrap(processErr, fmt.Sprintf("failed to process logging file %s", fileMeta.FileName)) } - fileMeta.Processed = true - now := time.Now() - fileMeta.ProcessedTime = &now - err = tx.Update(fileMeta) - if err != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + // Batch write to DB in a single transaction + tx := db.Begin() + for _, r := range results { + for _, chatLog := range r.ChatLogs { + if err := tx.CreateOrUpdate(chatLog); err != nil { + tx.Rollback() + return errors.Default.Wrap(err, "failed to save chat log") + } + } + for _, compLog := range r.CompLogs { + if err := tx.CreateOrUpdate(compLog); err != nil { + tx.Rollback() + return errors.Default.Wrap(err, "failed to save completion log") + } + } + // Mark file as processed + r.FileMeta.Processed = true + now := time.Now() + r.FileMeta.ProcessedTime = &now + if err := tx.Update(r.FileMeta); err != nil { + tx.Rollback() + return errors.Default.Wrap(err, "failed to update file metadata") } - return errors.Default.Wrap(err, "failed to update file metadata") } - - err = tx.Commit() - if err != nil { - return errors.Default.Wrap(err, "failed to commit transaction") + if err := tx.Commit(); err != nil { + return errors.Default.Wrap(err, "failed to commit batch") } - taskCtx.IncProgress(1) + taskCtx.IncProgress(len(batch)) } return nil } +// parallelDownloadAndParse downloads and parses S3 files concurrently +func parallelDownloadAndParse( + taskCtx plugin.SubTaskContext, + data *QDevTaskData, + fileMetas []*models.QDevS3FileMeta, + displayNameCache *sync.Map, +) []downloadResult { + results := make([]downloadResult, len(fileMetas)) + jobs := make(chan int, s3DownloadChanSize) + var wg sync.WaitGroup + + // Start workers + for w := 0; w < s3DownloadWorkers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for idx := range jobs { + fm := fileMetas[idx] + result := downloadAndParseFile(taskCtx, data, fm, displayNameCache) + results[idx] = result + } + }() + } + + // Send jobs + for i := range fileMetas { + jobs <- i + } + close(jobs) + wg.Wait() + + return results +} + +// downloadAndParseFile downloads one S3 file and parses it into model records +func downloadAndParseFile( + taskCtx plugin.SubTaskContext, + data *QDevTaskData, + fileMeta *models.QDevS3FileMeta, + displayNameCache *sync.Map, +) downloadResult { + result := downloadResult{FileMeta: fileMeta} + + getResult, err := data.S3Client.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(data.S3Client.Bucket), + Key: aws.String(fileMeta.S3Path), + }) + if err != nil { + result.Err = err + return result + } + defer getResult.Body.Close() + + gzReader, err := gzip.NewReader(getResult.Body) + if err != nil { + result.Err = err + return result + } + defer gzReader.Close() + + var logFile loggingFile + if err := json.NewDecoder(gzReader).Decode(&logFile); err != nil { + result.Err = err + return result + } + + isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse") + + for _, rawRecord := range logFile.Records { + if isChatLog { + chatLog, err := parseChatRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache) + if err != nil { + result.Err = err + return result + } + if chatLog != nil { + result.ChatLogs = append(result.ChatLogs, chatLog) + } + } else { + compLog, err := parseCompletionRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache) + if err != nil { + result.Err = err + return result + } + if compLog != nil { + result.CompLogs = append(result.CompLogs, compLog) + } + } + } + + return result +} + +// cachedResolveDisplayName resolves display name with caching +func cachedResolveDisplayName(userId string, identityClient UserDisplayNameResolver, cache *sync.Map) string { + if v, ok := cache.Load(userId); ok { + return v.(string) + } + if identityClient == nil { + cache.Store(userId, userId) + return userId + } + displayName, err := identityClient.ResolveUserDisplayName(userId) + if err != nil || displayName == "" { + cache.Store(userId, userId) + return userId + } + cache.Store(userId, displayName) + return displayName +} + // JSON structures for logging data type loggingFile struct { @@ -147,48 +294,14 @@ type completionLogResponse struct { Completions []json.RawMessage `json:"completions"` } -func processLoggingData(taskCtx plugin.SubTaskContext, db dal.Dal, reader io.ReadCloser, fileMeta *models.QDevS3FileMeta) errors.Error { - defer reader.Close() - - data := taskCtx.GetData().(*QDevTaskData) - - gzReader, err := gzip.NewReader(reader) - if err != nil { - return errors.Convert(err) - } - defer gzReader.Close() - - var logFile loggingFile - decoder := json.NewDecoder(gzReader) - if err := decoder.Decode(&logFile); err != nil { - return errors.Convert(err) - } - - isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse") - - for _, rawRecord := range logFile.Records { - if isChatLog { - if err := processChatRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { - return err - } - } else { - if err := processCompletionRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { - return err - } - } - } - - return nil -} - -func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { +func parseChatRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver, cache *sync.Map) (*models.QDevChatLog, error) { var record chatLogRecord if err := json.Unmarshal(raw, &record); err != nil { - return errors.Convert(err) + return nil, err } if record.Request == nil || record.Response == nil { - return nil + return nil, nil } ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) @@ -202,7 +315,7 @@ func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMe ScopeId: fileMeta.ScopeId, RequestId: record.Response.RequestID, UserId: userId, - DisplayName: resolveDisplayName(taskCtx.GetLogger(), userId, identityClient), + DisplayName: cachedResolveDisplayName(userId, identityClient, cache), Timestamp: ts, ChatTriggerType: record.Request.ChatTriggerType, HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", @@ -225,7 +338,7 @@ func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMe chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID } - return errors.Default.Wrap(db.CreateOrUpdate(chatLog), "failed to save chat log") + return chatLog, nil } // countOpenFiles counts tags within block @@ -253,7 +366,6 @@ func parseActiveFile(prompt string) (string, string) { return "", "" } block := prompt[start : start+end] - // Find nameStart := strings.Index(block, "name=\"") if nameStart == -1 { return "", "" @@ -268,14 +380,14 @@ func parseActiveFile(prompt string) (string, string) { return fileName, ext } -func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { +func parseCompletionRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver, cache *sync.Map) (*models.QDevCompletionLog, error) { var record completionLogRecord if err := json.Unmarshal(raw, &record); err != nil { - return errors.Convert(err) + return nil, err } if record.Request == nil || record.Response == nil { - return nil + return nil, nil } ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) @@ -284,25 +396,22 @@ func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json } userId := normalizeUserId(record.Request.UserID) - completionLog := &models.QDevCompletionLog{ + return &models.QDevCompletionLog{ ConnectionId: fileMeta.ConnectionId, ScopeId: fileMeta.ScopeId, RequestId: record.Response.RequestID, UserId: userId, - DisplayName: resolveDisplayName(taskCtx.GetLogger(), userId, identityClient), + DisplayName: cachedResolveDisplayName(userId, identityClient, cache), Timestamp: ts, FileName: record.Request.FileName, FileExtension: filepath.Ext(record.Request.FileName), HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", CompletionsCount: len(record.Response.Completions), - } - - return errors.Default.Wrap(db.CreateOrUpdate(completionLog), "failed to save completion log") + }, nil } // normalizeUserId strips the "d-{directoryId}." prefix from Identity Center user IDs // so that logging user IDs match the short UUID format used in user-report CSVs. -// e.g. "d-9067deb161.6478a4a8-60a1-70d9-37bc-6aae85f6746a" → "6478a4a8-60a1-70d9-37bc-6aae85f6746a" func normalizeUserId(userId string) string { if idx := strings.LastIndex(userId, "."); idx != -1 && strings.HasPrefix(userId, "d-") { return userId[idx+1:] From 392ed3c3bf7d64126740a23cea6f32dec93cd4cd Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 11:53:38 +0800 Subject: [PATCH 14/21] fix(q-dev): check tx.Rollback error return to satisfy errcheck lint --- .../q_dev/tasks/s3_logging_extractor.go | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go index 2b28edc0a50..df55a663b13 100644 --- a/backend/plugins/q_dev/tasks/s3_logging_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -105,27 +105,36 @@ func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { // Batch write to DB in a single transaction tx := db.Begin() + var txErr errors.Error for _, r := range results { for _, chatLog := range r.ChatLogs { - if err := tx.CreateOrUpdate(chatLog); err != nil { - tx.Rollback() - return errors.Default.Wrap(err, "failed to save chat log") + if txErr = tx.CreateOrUpdate(chatLog); txErr != nil { + break } } + if txErr != nil { + break + } for _, compLog := range r.CompLogs { - if err := tx.CreateOrUpdate(compLog); err != nil { - tx.Rollback() - return errors.Default.Wrap(err, "failed to save completion log") + if txErr = tx.CreateOrUpdate(compLog); txErr != nil { + break } } - // Mark file as processed + if txErr != nil { + break + } r.FileMeta.Processed = true now := time.Now() r.FileMeta.ProcessedTime = &now - if err := tx.Update(r.FileMeta); err != nil { - tx.Rollback() - return errors.Default.Wrap(err, "failed to update file metadata") + if txErr = tx.Update(r.FileMeta); txErr != nil { + break + } + } + if txErr != nil { + if rbErr := tx.Rollback(); rbErr != nil { + taskCtx.GetLogger().Error(rbErr, "failed to rollback transaction") } + return errors.Default.Wrap(txErr, "failed to write logging batch") } if err := tx.Commit(); err != nil { return errors.Default.Wrap(err, "failed to commit batch") From acf133b0e4ee10d2c245b15a5aeb71c67bfcc85d Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 14:12:29 +0800 Subject: [PATCH 15/21] feat(q-dev): add per-user model usage table and models column Add "Per-User Model Usage" table (panel 11) showing each user's request count and avg prompt/response length per model_id. Also add "Models Used" column to the Per-User Activity table. --- grafana/dashboards/qdev_logging.json | 64 +++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json index b6d0627a56e..8cecdc26cbc 100644 --- a/grafana/dashboards/qdev_logging.json +++ b/grafana/dashboards/qdev_logging.json @@ -541,7 +541,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", + "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.models_used as 'Models Used',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n GROUP_CONCAT(DISTINCT CASE WHEN model_id != '' AND model_id IS NOT NULL THEN model_id END ORDER BY model_id SEPARATOR ', ') as models_used,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", "refId": "A" } ], @@ -775,6 +775,68 @@ ], "title": "Prompt & Response Length Trends", "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Per-user breakdown of model usage in chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n CASE WHEN model_id = '' OR model_id IS NULL THEN '(unknown)' ELSE model_id END as 'Model',\n COUNT(*) as 'Requests',\n ROUND(AVG(prompt_length)) as 'Avg Prompt Len',\n ROUND(AVG(response_length)) as 'Avg Response Len'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id, model_id\nORDER BY user_id, COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Per-User Model Usage", + "type": "table" } ], "preload": false, From 9b71150ce8f8505da028a4916c65527b46241b40 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 14:14:30 +0800 Subject: [PATCH 16/21] fix(q-dev): remove per-user model usage table, keep models column only --- grafana/dashboards/qdev_logging.json | 64 +--------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json index 8cecdc26cbc..72eb0d0d2d4 100644 --- a/grafana/dashboards/qdev_logging.json +++ b/grafana/dashboards/qdev_logging.json @@ -775,68 +775,6 @@ ], "title": "Prompt & Response Length Trends", "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "Per-user breakdown of model usage in chat events", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 11, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n CASE WHEN model_id = '' OR model_id IS NULL THEN '(unknown)' ELSE model_id END as 'Model',\n COUNT(*) as 'Requests',\n ROUND(AVG(prompt_length)) as 'Avg Prompt Len',\n ROUND(AVG(response_length)) as 'Avg Response Len'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id, model_id\nORDER BY user_id, COUNT(*) DESC", - "refId": "A" - } - ], - "title": "Per-User Model Usage", - "type": "table" } ], "preload": false, @@ -859,4 +797,4 @@ "title": "Kiro AI Activity Insights", "uid": "qdev_logging", "version": 1 -} +} \ No newline at end of file From 9a5ade336d0163538a0bf8d83f554a8bd1510331 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 14:59:30 +0800 Subject: [PATCH 17/21] feat(q-dev): add Kiro Executive Dashboard with cross-source analytics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New dashboard "Kiro Executive Dashboard" with 12 panels covering: - KPIs: WAU, credits efficiency, acceptance rate, steering adoption - Trends: weekly active users, new vs returning users - Adoption funnel: Chat→Inline→CodeFix→Review→DocGen→TestGen→Agentic→Steering→Spec - Cost: credits pace vs projected monthly, idle power users - Quality: acceptance rate trends, code review findings, test generation - Efficiency: per-user productivity table with credits/line ratio Correlates data across user_report (credits), user_data (code metrics), and chat_log (interaction patterns) for holistic Kiro usage insights. --- grafana/dashboards/qdev_executive.json | 958 +++++++++++++++++++++++++ 1 file changed, 958 insertions(+) create mode 100644 grafana/dashboards/qdev_executive.json diff --git a/grafana/dashboards/qdev_executive.json b/grafana/dashboards/qdev_executive.json new file mode 100644 index 00000000000..6ef510855af --- /dev/null +++ b/grafana/dashboards/qdev_executive.json @@ -0,0 +1,958 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "mysql", + "description": "Distinct users with chat activity in the last 7 days", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", + "refId": "A" + } + ], + "title": "Weekly Active Users", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Average credits spent per accepted line of code", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.inline_ai_code_lines + d.chat_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 0), 2) as 'Credits per Accepted Line'\nFROM lake._tool_q_dev_user_report r\nJOIN lake._tool_q_dev_user_data d ON r.user_id = d.user_id AND r.date = d.date\nWHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits Efficiency", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Percentage of inline suggestions accepted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Inline Acceptance Rate", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Percentage of users using steering rules", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CONCAT(ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0), '%') as 'Users with Steering'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Adoption", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Weekly active user count over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(YEARWEEK(timestamp, 1), ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY YEARWEEK(timestamp, 1)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Active Users Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "New vs returning users by week", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n week as time,\n SUM(CASE WHEN is_new = 1 THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN is_new = 0 THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT\n u.user_id,\n STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') as week,\n CASE WHEN STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') = STR_TO_DATE(CONCAT(YEARWEEK(f.first_seen, 1), ' Monday'), '%X%V %W') THEN 1 ELSE 0 END as is_new\n FROM lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, MIN(timestamp) as first_seen FROM lake._tool_q_dev_chat_log GROUP BY user_id) f ON u.user_id = f.user_id\n WHERE $__timeFilter(u.timestamp)\n GROUP BY u.user_id, YEARWEEK(u.timestamp, 1), f.first_seen\n) weekly\nGROUP BY week\nORDER BY week", + "refId": "A" + } + ], + "title": "New vs Returning Users (Weekly)", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Number of users who used each feature", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "barRadius": 0.1, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0 + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Steering', COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)\nUNION ALL SELECT 'Spec Mode', COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Feature Adoption Funnel", + "type": "barchart" + }, + { + "datasource": "mysql", + "description": "Cumulative credits this month vs projected total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Credits Pace vs Projected (This Month)", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Acceptance Rate Trends", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Code review findings and test generation metrics over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_accepted_tests) as 'Tests Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Code Review Findings & Test Generation", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Per-user productivity and efficiency metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN lake._tool_q_dev_user_report r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", + "refId": "A" + } + ], + "title": "User Productivity & Efficiency", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Power tier users with no activity in the last 14 days", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "refId": "A" + } + ], + "title": "Idle Power Users (No Activity in 14 Days)", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "executive", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Executive Dashboard", + "uid": "qdev_executive", + "version": 1 +} \ No newline at end of file From 06f300b2a8e138b6ce0562681fc7e54ed1ace6eb Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 15:26:36 +0800 Subject: [PATCH 18/21] fix(q-dev): fix pie charts to show per-row slices instead of single total Set reduceOptions.values=true so Grafana treats each SQL result row as a separate pie slice. Fixes Model Usage Distribution, File Type Usage, Kiro Feature Adoption, and Active File Types pie charts. --- grafana/dashboards/qdev_logging.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json index 72eb0d0d2d4..59a6556da6b 100644 --- a/grafana/dashboards/qdev_logging.json +++ b/grafana/dashboards/qdev_logging.json @@ -210,10 +210,10 @@ "pieType": "donut", "reduceOptions": { "calcs": [ - "sum" + "lastNotNull" ], - "fields": "", - "values": false + "fields": "/^Requests$/", + "values": true }, "tooltip": { "mode": "single", @@ -278,10 +278,10 @@ "pieType": "pie", "reduceOptions": { "calcs": [ - "sum" + "lastNotNull" ], "fields": "", - "values": false + "values": true }, "tooltip": { "mode": "single", @@ -592,10 +592,10 @@ "pieType": "donut", "reduceOptions": { "calcs": [ - "sum" + "lastNotNull" ], "fields": "", - "values": false + "values": true }, "tooltip": { "mode": "single", @@ -660,10 +660,10 @@ "pieType": "pie", "reduceOptions": { "calcs": [ - "sum" + "lastNotNull" ], "fields": "", - "values": false + "values": true }, "tooltip": { "mode": "single", From 24d50c032699580692af46f31b1c730f443f072d Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 15:28:42 +0800 Subject: [PATCH 19/21] fix(q-dev): cast Hour to string for Active Hours bar chart x-axis --- grafana/dashboards/qdev_logging.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json index 59a6556da6b..462adcd986e 100644 --- a/grafana/dashboards/qdev_logging.json +++ b/grafana/dashboards/qdev_logging.json @@ -159,7 +159,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n hour_of_day as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", + "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", "refId": "A" } ], From 6b6caa78c248eacf912f6789d8c89e49dc9d8bdc Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 15:36:03 +0800 Subject: [PATCH 20/21] fix(q-dev): fix pie chart single-slice and GROUP BY display_name issues 1. qdev_user_report Panel 4 (Subscription Tier Distribution): set reduceOptions.values=true to show per-tier slices 2. qdev_user_data Panel 6 (User Interactions): remove display_name from GROUP BY, use MAX(display_name) to merge same user --- grafana/dashboards/qdev_user_data.json | 2 +- grafana/dashboards/qdev_user_report.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/dashboards/qdev_user_data.json b/grafana/dashboards/qdev_user_data.json index e66901a5dbf..578cf095dca 100644 --- a/grafana/dashboards/qdev_user_data.json +++ b/grafana/dashboards/qdev_user_data.json @@ -730,7 +730,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name\nORDER BY SUM(inline_ai_code_lines) DESC", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_ai_code_lines) DESC", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/qdev_user_report.json b/grafana/dashboards/qdev_user_report.json index 850a20fa1f4..920fc1f6cde 100644 --- a/grafana/dashboards/qdev_user_report.json +++ b/grafana/dashboards/qdev_user_report.json @@ -305,10 +305,10 @@ "pieType": "pie", "reduceOptions": { "calcs": [ - "sum" + "lastNotNull" ], "fields": "", - "values": false + "values": true }, "tooltip": { "mode": "single", @@ -461,4 +461,4 @@ "title": "Kiro Usage Dashboard", "uid": "qdev_user_report", "version": 1 -} +} \ No newline at end of file From 1edef6d61fca57daa27ad827a023e12545838c94 Mon Sep 17 00:00:00 2001 From: warren Date: Sun, 15 Mar 2026 15:44:49 +0800 Subject: [PATCH 21/21] fix(q-dev): prevent data inflation in user_report JOIN user_data user_report has multiple rows per (user_id, date) due to client_type (KIRO_IDE, KIRO_CLI), but user_data has only one row per (user_id, date). A direct JOIN causes user_data metrics to be counted multiple times. Fix: pre-aggregate user_report by (user_id, date) in a subquery before joining, so the JOIN is always 1:1. Affects: Credits Efficiency stat and User Productivity table. --- grafana/dashboards/qdev_executive.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grafana/dashboards/qdev_executive.json b/grafana/dashboards/qdev_executive.json index 6ef510855af..c6e2524d70b 100644 --- a/grafana/dashboards/qdev_executive.json +++ b/grafana/dashboards/qdev_executive.json @@ -129,7 +129,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.inline_ai_code_lines + d.chat_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 0), 2) as 'Credits per Accepted Line'\nFROM lake._tool_q_dev_user_report r\nJOIN lake._tool_q_dev_user_data d ON r.user_id = d.user_id AND r.date = d.date\nWHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n SELECT user_id, date, SUM(credits_used) as credits_used\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter(date)\n) d ON r.user_id = d.user_id AND r.date = d.date", "refId": "A" } ], @@ -865,7 +865,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN lake._tool_q_dev_user_report r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", + "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN (\n SELECT user_id, date, SUM(credits_used) as credits_used, MAX(subscription_tier) as subscription_tier\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", "refId": "A" } ],