From 4f753d6957a642ea519315f4e7793833c1401629 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:30:09 +0000 Subject: [PATCH 1/7] Initial plan From ff54ef73ba2f88459e432297d9e35c811364b131 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:50:38 +0000 Subject: [PATCH 2/7] Implement tool sequence graph generation with --tool-graph option Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs.go | 20 ++- pkg/cli/logs_test.go | 4 +- pkg/cli/tool_graph.go | 279 ++++++++++++++++++++++++++++++++++ pkg/cli/tool_graph_test.go | 97 ++++++++++++ pkg/workflow/claude_engine.go | 44 ++++-- pkg/workflow/codex_engine.go | 33 +++- pkg/workflow/metrics.go | 1 + 7 files changed, 454 insertions(+), 24 deletions(-) create mode 100644 pkg/cli/tool_graph.go create mode 100644 pkg/cli/tool_graph_test.go diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index 30d891b64b6..23a9951a5e3 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -132,7 +132,8 @@ Examples: ` + constants.CLIExtensionPrefix + ` logs --start-date -1mo # Filter runs from last month ` + constants.CLIExtensionPrefix + ` logs --engine claude # Filter logs by claude engine ` + constants.CLIExtensionPrefix + ` logs --engine codex # Filter logs by codex engine - ` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory`, + ` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory + ` + constants.CLIExtensionPrefix + ` logs --tool-graph # Generate Mermaid tool sequence graph`, Run: func(cmd *cobra.Command, args []string) { var workflowName string if len(args) > 0 && args[0] != "" { @@ -165,6 +166,7 @@ Examples: outputDir, _ := cmd.Flags().GetString("output") engine, _ := cmd.Flags().GetString("engine") verbose, _ := cmd.Flags().GetBool("verbose") + toolGraph, _ := cmd.Flags().GetBool("tool-graph") // Resolve relative dates to absolute dates for GitHub CLI now := time.Now() @@ -204,7 +206,7 @@ Examples: } } - if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, verbose); err != nil { + if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, verbose, toolGraph); err != nil { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", Message: err.Error(), @@ -221,12 +223,13 @@ Examples: logsCmd.Flags().StringP("output", "o", "./logs", "Output directory for downloaded logs and artifacts") logsCmd.Flags().String("engine", "", "Filter logs by agentic engine type (claude, codex)") logsCmd.Flags().BoolP("verbose", "v", false, "Show individual tool names instead of grouping by MCP server") + logsCmd.Flags().Bool("tool-graph", false, "Generate Mermaid tool sequence graph from agent logs") return logsCmd } // DownloadWorkflowLogs downloads and analyzes workflow logs with metrics -func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine string, verbose bool) error { +func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine string, verbose bool, toolGraph bool) error { if verbose { fmt.Println(console.FormatInfoMessage("Fetching workflow runs from GitHub Actions...")) } @@ -410,6 +413,11 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou // Display missing tools analysis displayMissingToolsAnalysis(processedRuns, verbose) + // Generate tool sequence graph if requested + if toolGraph { + generateToolGraph(processedRuns, verbose) + } + // Display logs location prominently absOutputDir, _ := filepath.Abs(outputDir) fmt.Println(console.FormatSuccessMessage(fmt.Sprintf("Downloaded %d logs to %s", len(processedRuns), absOutputDir))) @@ -970,7 +978,7 @@ func displayToolCallReport(processedRuns []ProcessedRun, verbose bool) { // For now, let's extract metrics from the run if available // We'll process log files to get tool call information - logMetrics := extractLogMetricsFromRun(processedRun) + logMetrics := ExtractLogMetricsFromRun(processedRun) for _, toolCall := range logMetrics.ToolCalls { var displayKey string @@ -1060,8 +1068,8 @@ func displayToolCallReport(processedRuns []ProcessedRun, verbose bool) { fmt.Print(console.RenderTable(tableConfig)) } -// extractLogMetricsFromRun extracts log metrics from a processed run's log directory -func extractLogMetricsFromRun(processedRun ProcessedRun) workflow.LogMetrics { +// ExtractLogMetricsFromRun extracts log metrics from a processed run's log directory +func ExtractLogMetricsFromRun(processedRun ProcessedRun) workflow.LogMetrics { // Use the LogsPath from the WorkflowRun to get metrics if processedRun.Run.LogsPath == "" { return workflow.LogMetrics{} diff --git a/pkg/cli/logs_test.go b/pkg/cli/logs_test.go index 2761ca92de2..dbe48e576a2 100644 --- a/pkg/cli/logs_test.go +++ b/pkg/cli/logs_test.go @@ -17,7 +17,7 @@ func TestDownloadWorkflowLogs(t *testing.T) { // Test the DownloadWorkflowLogs function // This should either fail with auth error (if not authenticated) // or succeed with no results (if authenticated but no workflows match) - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", false) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", false, false) // If GitHub CLI is authenticated, the function may succeed but find no results // If not authenticated, it should return an auth error @@ -793,7 +793,7 @@ func TestDownloadWorkflowLogsWithEngineFilter(t *testing.T) { if !tt.expectError { // For valid engines, test that the function can be called without panic // It may still fail with auth errors, which is expected - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, false) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, false, false) // Clean up any created directories os.RemoveAll("./test-logs") diff --git a/pkg/cli/tool_graph.go b/pkg/cli/tool_graph.go new file mode 100644 index 00000000000..a92a1d3adf7 --- /dev/null +++ b/pkg/cli/tool_graph.go @@ -0,0 +1,279 @@ +package cli + +import ( + "fmt" + "sort" + "strings" + + "github.com/githubnext/gh-aw/pkg/console" +) + +// ToolTransition represents an edge in the tool graph +type ToolTransition struct { + From string // Source tool name + To string // Target tool name + Count int // Number of times this transition occurred +} + +// ToolGraph represents a directed graph of tool call sequences +type ToolGraph struct { + Tools map[string]bool // Set of all tools + Transitions map[string]int // Key: "from->to", Value: count + sequences [][]string // Store sequences for analysis +} + +// NewToolGraph creates a new empty tool graph +func NewToolGraph() *ToolGraph { + return &ToolGraph{ + Tools: make(map[string]bool), + Transitions: make(map[string]int), + sequences: make([][]string, 0), + } +} + +// AddSequence adds a tool call sequence to the graph +func (g *ToolGraph) AddSequence(tools []string) { + if len(tools) == 0 { + return + } + + // Add all tools to the set + for _, tool := range tools { + g.Tools[tool] = true + } + + // Add transitions between consecutive tools + for i := 0; i < len(tools)-1; i++ { + from := tools[i] + to := tools[i+1] + key := fmt.Sprintf("%s->%s", from, to) + g.Transitions[key]++ + } + + // Store the sequence for analysis + g.sequences = append(g.sequences, tools) +} + +// GenerateMermaidGraph generates a Mermaid state diagram from the tool graph +func (g *ToolGraph) GenerateMermaidGraph() string { + if len(g.Tools) == 0 { + return "```mermaid\nstateDiagram-v2\n [*] --> [*] : No tool calls found\n```\n" + } + + var sb strings.Builder + sb.WriteString("```mermaid\n") + sb.WriteString("stateDiagram-v2\n") + + // Add tool states with normalized names for Mermaid + toolToStateMap := make(map[string]string) + var tools []string + for tool := range g.Tools { + tools = append(tools, tool) + } + sort.Strings(tools) + + for i, tool := range tools { + stateId := fmt.Sprintf("tool%d", i) + toolToStateMap[tool] = stateId + // Escape special characters in tool names for display + displayName := strings.ReplaceAll(tool, "\"", "\\\"") + sb.WriteString(fmt.Sprintf(" %s : %s\n", stateId, displayName)) + } + + // Add start state + sb.WriteString(" [*] --> start_tool : begin\n") + + // Find the most common starting tool(s) + startCounts := make(map[string]int) + for _, sequence := range g.sequences { + if len(sequence) > 0 { + startCounts[sequence[0]]++ + } + } + + // Connect start to the most common starting tools + if len(startCounts) > 0 { + var startTools []string + maxCount := 0 + for tool, count := range startCounts { + if count > maxCount { + maxCount = count + startTools = []string{tool} + } else if count == maxCount { + startTools = append(startTools, tool) + } + } + + for _, tool := range startTools { + if stateId, exists := toolToStateMap[tool]; exists { + sb.WriteString(fmt.Sprintf(" start_tool --> %s\n", stateId)) + } + } + } + + // Add transitions with counts as labels + var transitions []ToolTransition + for key, count := range g.Transitions { + parts := strings.Split(key, "->") + if len(parts) == 2 { + transitions = append(transitions, ToolTransition{ + From: parts[0], + To: parts[1], + Count: count, + }) + } + } + + // Sort transitions by count (descending) for better visualization + sort.Slice(transitions, func(i, j int) bool { + if transitions[i].Count != transitions[j].Count { + return transitions[i].Count > transitions[j].Count + } + if transitions[i].From != transitions[j].From { + return transitions[i].From < transitions[j].From + } + return transitions[i].To < transitions[j].To + }) + + for _, transition := range transitions { + fromState, fromExists := toolToStateMap[transition.From] + toState, toExists := toolToStateMap[transition.To] + + if fromExists && toExists { + label := "" + if transition.Count > 1 { + label = fmt.Sprintf(" : %dx", transition.Count) + } + sb.WriteString(fmt.Sprintf(" %s --> %s%s\n", fromState, toState, label)) + } + } + + sb.WriteString("```\n") + return sb.String() +} + +// GetSummary returns a summary of the tool graph +func (g *ToolGraph) GetSummary() string { + if len(g.Tools) == 0 { + return "No tool sequences found in the logs." + } + + var sb strings.Builder + sb.WriteString("🔄 Tool Sequence Graph Summary\n") + sb.WriteString(fmt.Sprintf(" • %d unique tools\n", len(g.Tools))) + sb.WriteString(fmt.Sprintf(" • %d tool transitions\n", len(g.Transitions))) + sb.WriteString(fmt.Sprintf(" • %d sequences analyzed\n", len(g.sequences))) + + // Find most common transitions + if len(g.Transitions) > 0 { + var topTransitions []ToolTransition + for key, count := range g.Transitions { + parts := strings.Split(key, "->") + if len(parts) == 2 { + topTransitions = append(topTransitions, ToolTransition{ + From: parts[0], + To: parts[1], + Count: count, + }) + } + } + + sort.Slice(topTransitions, func(i, j int) bool { + return topTransitions[i].Count > topTransitions[j].Count + }) + + sb.WriteString("\nMost common tool transitions:\n") + for i, transition := range topTransitions { + if i >= 5 { // Show top 5 + break + } + sb.WriteString(fmt.Sprintf(" %d. %s → %s (%dx)\n", + i+1, transition.From, transition.To, transition.Count)) + } + } + + return sb.String() +} + +// generateToolGraph analyzes processed runs and generates a tool sequence graph +func generateToolGraph(processedRuns []ProcessedRun, verbose bool) { + if len(processedRuns) == 0 { + return + } + + if verbose { + fmt.Println(console.FormatInfoMessage("Analyzing tool call sequences for graph generation...")) + } + + graph := NewToolGraph() + + // Extract tool sequences from each run + for _, run := range processedRuns { + sequences := extractToolSequencesFromRun(run, verbose) + for _, sequence := range sequences { + graph.AddSequence(sequence) + } + } + + // Display summary + fmt.Printf("\n%s\n", console.FormatListHeader("🔄 Tool Sequence Analysis")) + fmt.Printf("%s\n", console.FormatListHeader("=========================")) + fmt.Println(graph.GetSummary()) + + // Generate and display Mermaid graph + fmt.Printf("\n%s\n", console.FormatListHeader("📊 Mermaid Tool Sequence Graph")) + fmt.Printf("%s\n", console.FormatListHeader("===============================")) + mermaidGraph := graph.GenerateMermaidGraph() + fmt.Println(mermaidGraph) + + if verbose { + fmt.Println(console.FormatSuccessMessage("Tool sequence graph generated successfully")) + } +} + +// extractToolSequencesFromRun extracts tool call sequences from a single run +func extractToolSequencesFromRun(run ProcessedRun, verbose bool) [][]string { + var sequences [][]string + + if run.Run.LogsPath == "" { + return sequences + } + + // Extract metrics from the run (which now includes tool sequences) + metrics := ExtractLogMetricsFromRun(run) + + // Use the tool sequences from the metrics if available + if len(metrics.ToolSequences) > 0 { + sequences = append(sequences, metrics.ToolSequences...) + + if verbose { + totalTools := 0 + for _, seq := range metrics.ToolSequences { + totalTools += len(seq) + } + fmt.Println(console.FormatInfoMessage(fmt.Sprintf("Extracted %d tool sequences with %d total tool calls from run %d", + len(metrics.ToolSequences), totalTools, run.Run.DatabaseID))) + } + } else if len(metrics.ToolCalls) > 0 { + // Fallback: convert tool calls to a simple sequence if no sequences available + // This provides some graph data even when sequence extraction fails + var tools []string + for _, toolCall := range metrics.ToolCalls { + // Add each tool based on its call count to approximate sequence + for i := 0; i < toolCall.CallCount; i++ { + tools = append(tools, toolCall.Name) + } + } + + if len(tools) > 0 { + sequences = append(sequences, tools) + } + + if verbose && len(tools) > 0 { + fmt.Println(console.FormatWarningMessage(fmt.Sprintf("No tool sequences found, using fallback with %d tool calls from run %d", + len(tools), run.Run.DatabaseID))) + } + } + + return sequences +} diff --git a/pkg/cli/tool_graph_test.go b/pkg/cli/tool_graph_test.go new file mode 100644 index 00000000000..6c539fd6ac8 --- /dev/null +++ b/pkg/cli/tool_graph_test.go @@ -0,0 +1,97 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestToolGraph(t *testing.T) { + // Test basic tool graph creation and Mermaid generation + graph := NewToolGraph() + + // Test empty graph + mermaid := graph.GenerateMermaidGraph() + if mermaid == "" { + t.Error("Expected non-empty Mermaid graph for empty tool graph") + } + + // Test with a simple sequence + sequence := []string{"bash_ls", "github_search_issues", "bash_cat"} + graph.AddSequence(sequence) + + if len(graph.Tools) != 3 { + t.Errorf("Expected 3 tools, got %d", len(graph.Tools)) + } + + if len(graph.Transitions) != 2 { + t.Errorf("Expected 2 transitions, got %d", len(graph.Transitions)) + } + + // Test Mermaid generation with actual data + mermaid = graph.GenerateMermaidGraph() + if mermaid == "" { + t.Error("Expected non-empty Mermaid graph") + } + + // Should contain mermaid syntax + if !strings.Contains(mermaid, "```mermaid") { + t.Error("Expected Mermaid graph to contain mermaid code block") + } + + if !strings.Contains(mermaid, "stateDiagram-v2") { + t.Error("Expected Mermaid graph to use stateDiagram-v2 syntax") + } + + // Test summary generation + summary := graph.GetSummary() + if summary == "" { + t.Error("Expected non-empty summary") + } +} + +func TestToolGraphMultipleSequences(t *testing.T) { + graph := NewToolGraph() + + // Add multiple sequences + seq1 := []string{"bash_ls", "github_search_issues"} + seq2 := []string{"bash_ls", "bash_cat"} + seq3 := []string{"github_search_issues", "bash_cat"} + + graph.AddSequence(seq1) + graph.AddSequence(seq2) + graph.AddSequence(seq3) + + // Should have 3 unique tools + if len(graph.Tools) != 3 { + t.Errorf("Expected 3 unique tools, got %d", len(graph.Tools)) + } + + // Should have transitions with counts + expectedTransitions := map[string]int{ + "bash_ls->github_search_issues": 1, + "bash_ls->bash_cat": 1, + "github_search_issues->bash_cat": 1, + } + + for key, expectedCount := range expectedTransitions { + if actualCount, exists := graph.Transitions[key]; !exists || actualCount != expectedCount { + t.Errorf("Expected transition %s to have count %d, got %d", key, expectedCount, actualCount) + } + } +} + +func TestToolGraphEmptySequences(t *testing.T) { + graph := NewToolGraph() + + // Add empty sequence + graph.AddSequence([]string{}) + + // Should remain empty + if len(graph.Tools) != 0 { + t.Errorf("Expected 0 tools for empty sequence, got %d", len(graph.Tools)) + } + + if len(graph.Transitions) != 0 { + t.Errorf("Expected 0 transitions for empty sequence, got %d", len(graph.Transitions)) + } +} diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index d75552292c0..9d009a232ea 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -748,6 +748,7 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe // Look for the result entry with type: "result" toolCallMap := make(map[string]*ToolCallInfo) // Track tool calls across entries + var currentSequence []string // Track tool sequence within current context for _, entry := range logEntries { if entryType, exists := entry["type"]; exists { @@ -786,16 +787,17 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) } break - } - } - - // Parse tool_use entries for tool call statistics - if entry["type"] == "user" { - if message, exists := entry["message"]; exists { - if messageMap, ok := message.(map[string]interface{}); ok { - if content, exists := messageMap["content"]; exists { - if contentArray, ok := content.([]interface{}); ok { - e.parseToolCalls(contentArray, toolCallMap) + } else if typeStr == "assistant" { + // Parse tool_use entries for tool call statistics and sequence + if message, exists := entry["message"]; exists { + if messageMap, ok := message.(map[string]interface{}); ok { + if content, exists := messageMap["content"]; exists { + if contentArray, ok := content.([]interface{}); ok { + sequenceInMessage := e.parseToolCallsWithSequence(contentArray, toolCallMap) + if len(sequenceInMessage) > 0 { + currentSequence = append(currentSequence, sequenceInMessage...) + } + } } } } @@ -803,6 +805,11 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe } } + // Add the complete sequence if we found any tool calls + if len(currentSequence) > 0 { + metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) + } + // Convert tool call map to slice for _, toolInfo := range toolCallMap { metrics.ToolCalls = append(metrics.ToolCalls, *toolInfo) @@ -816,8 +823,10 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe return metrics } -// parseToolCalls extracts tool call information from Claude log content array -func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap map[string]*ToolCallInfo) { +// parseToolCallsWithSequence extracts tool call information from Claude log content array and returns sequence +func (e *ClaudeEngine) parseToolCallsWithSequence(contentArray []interface{}, toolCallMap map[string]*ToolCallInfo) []string { + var sequence []string + for _, contentItem := range contentArray { if contentMap, ok := contentItem.(map[string]interface{}); ok { if contentType, exists := contentMap["type"]; exists { @@ -851,6 +860,9 @@ func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap ma } } + // Add to sequence + sequence = append(sequence, prettifiedName) + // Initialize or update tool call info if toolInfo, exists := toolCallMap[prettifiedName]; exists { toolInfo.CallCount++ @@ -888,6 +900,14 @@ func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap ma } } } + + return sequence +} + +// parseToolCalls extracts tool call information from Claude log content array (legacy method) +func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap map[string]*ToolCallInfo) { + // Use the new method but ignore the sequence return value for backward compatibility + e.parseToolCallsWithSequence(contentArray, toolCallMap) } // shortenCommand creates a short identifier for bash commands diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 7c5800f8de4..791a0088655 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -177,6 +177,7 @@ func (e *CodexEngine) ParseLogMetrics(logContent string, verbose bool) LogMetric turns := 0 inThinkingSection := false toolCallMap := make(map[string]*ToolCallInfo) // Track tool calls + var currentSequence []string // Track tool sequence for _, line := range lines { // Skip empty lines @@ -189,13 +190,20 @@ func (e *CodexEngine) ParseLogMetrics(logContent string, verbose bool) LogMetric if !inThinkingSection { turns++ inThinkingSection = true + // Start of a new thinking section, save previous sequence if any + if len(currentSequence) > 0 { + metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) + currentSequence = []string{} + } } } else if strings.Contains(line, "] tool") || strings.Contains(line, "] exec") || strings.Contains(line, "] codex") { inThinkingSection = false } - // Extract tool calls from Codex logs - e.parseCodexToolCalls(line, toolCallMap) + // Extract tool calls from Codex logs and add to sequence + if toolName := e.parseCodexToolCallsWithSequence(line, toolCallMap); toolName != "" { + currentSequence = append(currentSequence, toolName) + } // Extract Codex-specific token usage (always sum for Codex) if tokenUsage := e.extractCodexTokenUsage(line); tokenUsage > 0 { @@ -212,6 +220,11 @@ func (e *CodexEngine) ParseLogMetrics(logContent string, verbose bool) LogMetric } } + // Add final sequence if any + if len(currentSequence) > 0 { + metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) + } + metrics.TokenUsage = totalTokenUsage metrics.Turns = turns @@ -228,8 +241,8 @@ func (e *CodexEngine) ParseLogMetrics(logContent string, verbose bool) LogMetric return metrics } -// parseCodexToolCalls extracts tool call information from Codex log lines -func (e *CodexEngine) parseCodexToolCalls(line string, toolCallMap map[string]*ToolCallInfo) { +// parseCodexToolCallsWithSequence extracts tool call information from Codex log lines and returns tool name +func (e *CodexEngine) parseCodexToolCallsWithSequence(line string, toolCallMap map[string]*ToolCallInfo) string { // Parse tool calls: "] tool provider.method(...)" if strings.Contains(line, "] tool ") && strings.Contains(line, "(") { if match := regexp.MustCompile(`\] tool ([^(]+)\(`).FindStringSubmatch(line); len(match) > 1 { @@ -256,6 +269,8 @@ func (e *CodexEngine) parseCodexToolCalls(line string, toolCallMap map[string]*T MaxOutputSize: 0, // TODO: Extract output size from results if available } } + + return prettifiedName } } @@ -276,8 +291,18 @@ func (e *CodexEngine) parseCodexToolCalls(line string, toolCallMap map[string]*T MaxOutputSize: 0, } } + + return uniqueBashName } } + + return "" // No tool call found +} + +// parseCodexToolCalls extracts tool call information from Codex log lines (legacy method) +func (e *CodexEngine) parseCodexToolCalls(line string, toolCallMap map[string]*ToolCallInfo) { + // Use the new method but ignore the return value for backward compatibility + e.parseCodexToolCallsWithSequence(line, toolCallMap) } // shortenCommand creates a short identifier for bash commands diff --git a/pkg/workflow/metrics.go b/pkg/workflow/metrics.go index 511bdb18cf5..eedaf29bf51 100644 --- a/pkg/workflow/metrics.go +++ b/pkg/workflow/metrics.go @@ -23,6 +23,7 @@ type LogMetrics struct { WarningCount int Turns int // Number of turns needed to complete the task ToolCalls []ToolCallInfo // Tool call statistics + ToolSequences [][]string // Sequences of tool calls preserving order // Timestamp removed - use GitHub API timestamps instead of parsing from logs } From 9f7afd681e46c7041d41edc64863df56369dc13a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:52:30 +0000 Subject: [PATCH 3/7] Clean up unused legacy methods and complete tool graph implementation Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/claude_engine.go | 6 ------ pkg/workflow/codex_engine.go | 6 ------ pkg/workflow/js/collect_ndjson_output.test.cjs | 14 +++++++------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 9d009a232ea..1b63cbace31 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -904,12 +904,6 @@ func (e *ClaudeEngine) parseToolCallsWithSequence(contentArray []interface{}, to return sequence } -// parseToolCalls extracts tool call information from Claude log content array (legacy method) -func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap map[string]*ToolCallInfo) { - // Use the new method but ignore the sequence return value for backward compatibility - e.parseToolCallsWithSequence(contentArray, toolCallMap) -} - // shortenCommand creates a short identifier for bash commands func (e *ClaudeEngine) shortenCommand(command string) string { // Take first 20 characters and remove newlines diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 791a0088655..f233d22ad39 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -299,12 +299,6 @@ func (e *CodexEngine) parseCodexToolCallsWithSequence(line string, toolCallMap m return "" // No tool call found } -// parseCodexToolCalls extracts tool call information from Codex log lines (legacy method) -func (e *CodexEngine) parseCodexToolCalls(line string, toolCallMap map[string]*ToolCallInfo) { - // Use the new method but ignore the return value for backward compatibility - e.parseCodexToolCallsWithSequence(line, toolCallMap) -} - // shortenCommand creates a short identifier for bash commands func (e *CodexEngine) shortenCommand(command string) string { // Take first 20 characters and remove newlines diff --git a/pkg/workflow/js/collect_ndjson_output.test.cjs b/pkg/workflow/js/collect_ndjson_output.test.cjs index 50aa970599c..d7924369738 100644 --- a/pkg/workflow/js/collect_ndjson_output.test.cjs +++ b/pkg/workflow/js/collect_ndjson_output.test.cjs @@ -154,7 +154,7 @@ describe("collect_ndjson_output.cjs", () => { const failedMessage = mockCore.setFailed.mock.calls[0][0]; expect(failedMessage).toContain("requires a 'body' string field"); expect(failedMessage).toContain("requires a 'title' string field"); - + // setOutput should not be called because of early return const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); @@ -650,7 +650,7 @@ Line 3"} expect(mockCore.setFailed).toHaveBeenCalledTimes(1); const failedMessage = mockCore.setFailed.mock.calls[0][0]; expect(failedMessage).toContain("JSON parsing failed"); - + // setOutput should not be called because of early return const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); @@ -736,7 +736,7 @@ Line 3"} // Check if repair succeeded by looking at mock calls const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); - + if (outputCall) { // Repair succeeded const parsedOutput = JSON.parse(outputCall[1]); @@ -830,7 +830,7 @@ Line 3"} expect(mockCore.setFailed).toHaveBeenCalledTimes(1); const failedMessage = mockCore.setFailed.mock.calls[0][0]; expect(failedMessage).toContain("JSON parsing failed"); - + // setOutput should not be called because of early return const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); @@ -949,7 +949,7 @@ Line 3"} expect(mockCore.setFailed).toHaveBeenCalledTimes(1); const failedMessage = mockCore.setFailed.mock.calls[0][0]; expect(failedMessage).toContain("JSON parsing failed"); - + // setOutput should not be called because of early return const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); @@ -970,7 +970,7 @@ Line 3"} expect(mockCore.setFailed).toHaveBeenCalledTimes(1); const failedMessage = mockCore.setFailed.mock.calls[0][0]; expect(failedMessage).toContain("JSON parsing failed"); - + // setOutput should not be called because of early return const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); @@ -1015,7 +1015,7 @@ Line 3"} // Check if repair succeeded by looking at mock calls const setOutputCalls = mockCore.setOutput.mock.calls; const outputCall = setOutputCalls.find(call => call[0] === "output"); - + if (outputCall) { // Repair succeeded const parsedOutput = JSON.parse(outputCall[1]); From b0b00e1603d2ae97eb8bf4323611cd1c7e9e8b04 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Sep 2025 14:07:17 +0000 Subject: [PATCH 4/7] Remove summary and title from tool graph output, show only Mermaid graph Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/tool_graph.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/cli/tool_graph.go b/pkg/cli/tool_graph.go index a92a1d3adf7..b5a2c4ddad9 100644 --- a/pkg/cli/tool_graph.go +++ b/pkg/cli/tool_graph.go @@ -215,14 +215,7 @@ func generateToolGraph(processedRuns []ProcessedRun, verbose bool) { } } - // Display summary - fmt.Printf("\n%s\n", console.FormatListHeader("🔄 Tool Sequence Analysis")) - fmt.Printf("%s\n", console.FormatListHeader("=========================")) - fmt.Println(graph.GetSummary()) - - // Generate and display Mermaid graph - fmt.Printf("\n%s\n", console.FormatListHeader("📊 Mermaid Tool Sequence Graph")) - fmt.Printf("%s\n", console.FormatListHeader("===============================")) + // Generate and display Mermaid graph only mermaidGraph := graph.GenerateMermaidGraph() fmt.Println(mermaidGraph) From ce5a8e7e4f55fd3f2be70a311afd1a8366e9c09f Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 10 Sep 2025 15:15:44 +0000 Subject: [PATCH 5/7] Improve messages for empty access log and tool graph scenarios --- pkg/cli/access_log.go | 2 +- pkg/cli/tool_graph.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cli/access_log.go b/pkg/cli/access_log.go index d5fd84c7d39..341298250c0 100644 --- a/pkg/cli/access_log.go +++ b/pkg/cli/access_log.go @@ -268,7 +268,7 @@ func displayAccessLogAnalysis(processedRuns []ProcessedRun, verbose bool) { } if len(analyses) == 0 { - fmt.Println(console.FormatInfoMessage("No access logs found in downloaded runs")) + fmt.Println(console.FormatInfoMessage("No access logs found")) return } diff --git a/pkg/cli/tool_graph.go b/pkg/cli/tool_graph.go index b5a2c4ddad9..fa91007967d 100644 --- a/pkg/cli/tool_graph.go +++ b/pkg/cli/tool_graph.go @@ -57,7 +57,7 @@ func (g *ToolGraph) AddSequence(tools []string) { // GenerateMermaidGraph generates a Mermaid state diagram from the tool graph func (g *ToolGraph) GenerateMermaidGraph() string { if len(g.Tools) == 0 { - return "```mermaid\nstateDiagram-v2\n [*] --> [*] : No tool calls found\n```\n" + return console.FormatInfoMessage("No tool calls found") } var sb strings.Builder From 165c01541b14126ac615c791a7f633a3f4d2f55b Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 10 Sep 2025 16:05:56 +0000 Subject: [PATCH 6/7] Enhance log processing and tool graph generation by excluding specific output artifacts and improving verbose output for tool sequences --- debug_json.go | 95 +++++++++++++++++++++++++++++++++++ pkg/cli/logs.go | 34 +++++-------- pkg/cli/tool_graph.go | 18 +++---- pkg/workflow/claude_engine.go | 26 +++++++--- 4 files changed, 134 insertions(+), 39 deletions(-) create mode 100644 debug_json.go diff --git a/debug_json.go b/debug_json.go new file mode 100644 index 00000000000..e5182dd7b1b --- /dev/null +++ b/debug_json.go @@ -0,0 +1,95 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" +) + +func main() { + if len(os.Args) < 2 { + fmt.Println("Usage: go run debug_json.go ") + return + } + + filePath := os.Args[1] + content, err := os.ReadFile(filePath) + if err != nil { + fmt.Printf("Error reading file: %v\n", err) + return + } + + fmt.Printf("File size: %d bytes\n", len(content)) + + // Try to parse as JSON array + var logEntries []map[string]interface{} + if err := json.Unmarshal(content, &logEntries); err != nil { + fmt.Printf("Failed to parse as JSON array: %v\n", err) + + // Show first 200 characters + first200 := content + if len(first200) > 200 { + first200 = first200[:200] + } + fmt.Printf("First 200 chars: %q\n", string(first200)) + + // Show last 200 characters + last200 := content + if len(last200) > 200 { + last200 = last200[len(last200)-200:] + } + fmt.Printf("Last 200 chars: %q\n", string(last200)) + return + } + + fmt.Printf("Successfully parsed %d log entries\n", len(logEntries)) + + // Look at the structure of entries + assistantCount := 0 + resultCount := 0 + toolCallCount := 0 + + for i, entry := range logEntries { + if entryType, exists := entry["type"]; exists { + if typeStr, ok := entryType.(string); ok { + if typeStr == "result" { + resultCount++ + fmt.Printf("Entry %d: type=result\n", i) + } else if typeStr == "assistant" { + assistantCount++ + fmt.Printf("Entry %d: type=assistant", i) + if message, exists := entry["message"]; exists { + if messageMap, ok := message.(map[string]interface{}); ok { + if content, exists := messageMap["content"]; exists { + if contentArray, ok := content.([]interface{}); ok { + fmt.Printf(" (content array length: %d)", len(contentArray)) + for _, contentItem := range contentArray { + if contentMap, ok := contentItem.(map[string]interface{}); ok { + if contentType, exists := contentMap["type"]; exists { + if typeStr, ok := contentType.(string); ok { + if typeStr == "tool_use" { + toolCallCount++ + if name, exists := contentMap["name"]; exists { + fmt.Printf(" [tool_use: %v]", name) + } + } else { + fmt.Printf(" [%s]", typeStr) + } + } + } + } + } + } + } + } + } + fmt.Printf("\n") + } else { + fmt.Printf("Entry %d: type=%s\n", i, typeStr) + } + } + } + } + fmt.Printf("\nSummary: %d assistant entries, %d result entries, %d tool calls\n", + assistantCount, resultCount, toolCallCount) +} diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index 23a9951a5e3..53a882ba308 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -739,10 +739,12 @@ func extractLogMetrics(logDir string, verbose bool) (LogMetrics, error) { return nil } - // Process log files - if strings.HasSuffix(strings.ToLower(info.Name()), ".log") || - strings.HasSuffix(strings.ToLower(info.Name()), ".txt") || - strings.Contains(strings.ToLower(info.Name()), "log") { + // Process log files - exclude output artifacts like aw_output.txt + fileName := strings.ToLower(info.Name()) + if (strings.HasSuffix(fileName, ".log") || + (strings.HasSuffix(fileName, ".txt") && strings.Contains(fileName, "log"))) && + !strings.Contains(fileName, "aw_output") && + !strings.Contains(fileName, "agent_output") { fileMetrics, err := parseLogFileWithEngine(path, detectedEngine, verbose) if err != nil && verbose { @@ -760,6 +762,10 @@ func extractLogMetrics(logDir string, verbose bool) (LogMetrics, error) { // the total conversation turns for the entire workflow run metrics.Turns = fileMetrics.Turns } + + // Aggregate tool sequences and tool calls + metrics.ToolSequences = append(metrics.ToolSequences, fileMetrics.ToolSequences...) + metrics.ToolCalls = append(metrics.ToolCalls, fileMetrics.ToolCalls...) } return nil @@ -832,24 +838,10 @@ func extractEngineFromAwInfo(infoFilePath string, verbose bool) workflow.CodingA // parseLogFileWithEngine parses a log file using a specific engine or falls back to auto-detection func parseLogFileWithEngine(filePath string, detectedEngine workflow.CodingAgentEngine, verbose bool) (LogMetrics, error) { - // Read the log file content - file, err := os.Open(filePath) + // Read the entire log file at once to avoid JSON parsing issues from chunked reading + content, err := os.ReadFile(filePath) if err != nil { - return LogMetrics{}, fmt.Errorf("error opening log file: %w", err) - } - defer file.Close() - - var content []byte - buffer := make([]byte, 4096) - for { - n, err := file.Read(buffer) - if err != nil && err != io.EOF { - return LogMetrics{}, fmt.Errorf("error reading log file: %w", err) - } - if n == 0 { - break - } - content = append(content, buffer[:n]...) + return LogMetrics{}, fmt.Errorf("error reading log file: %w", err) } logContent := string(content) diff --git a/pkg/cli/tool_graph.go b/pkg/cli/tool_graph.go index fa91007967d..ac6f73eda6a 100644 --- a/pkg/cli/tool_graph.go +++ b/pkg/cli/tool_graph.go @@ -201,15 +201,17 @@ func generateToolGraph(processedRuns []ProcessedRun, verbose bool) { return } - if verbose { - fmt.Println(console.FormatInfoMessage("Analyzing tool call sequences for graph generation...")) - } - graph := NewToolGraph() - - // Extract tool sequences from each run for _, run := range processedRuns { sequences := extractToolSequencesFromRun(run, verbose) + if verbose && len(sequences) > 0 { + totalTools := 0 + for _, seq := range sequences { + totalTools += len(seq) + } + fmt.Println(console.FormatInfoMessage(fmt.Sprintf("Run %d contributed %d tool sequences with %d total tools", + run.Run.DatabaseID, len(sequences), totalTools))) + } for _, sequence := range sequences { graph.AddSequence(sequence) } @@ -218,10 +220,6 @@ func generateToolGraph(processedRuns []ProcessedRun, verbose bool) { // Generate and display Mermaid graph only mermaidGraph := graph.GenerateMermaidGraph() fmt.Println(mermaidGraph) - - if verbose { - fmt.Println(console.FormatSuccessMessage("Tool sequence graph generated successfully")) - } } // extractToolSequencesFromRun extracts tool call sequences from a single run diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 1b63cbace31..37207112983 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -614,7 +614,8 @@ func (e *ClaudeEngine) ParseLogMetrics(logContent string, verbose bool) LogMetri metrics.TokenUsage = resultMetrics.TokenUsage metrics.EstimatedCost = resultMetrics.EstimatedCost metrics.Turns = resultMetrics.Turns - metrics.ToolCalls = resultMetrics.ToolCalls // Copy tool calls as well + metrics.ToolCalls = resultMetrics.ToolCalls // Copy tool calls + metrics.ToolSequences = resultMetrics.ToolSequences // Copy tool sequences } } @@ -810,6 +811,15 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) } + if verbose && len(metrics.ToolSequences) > 0 { + totalTools := 0 + for _, seq := range metrics.ToolSequences { + totalTools += len(seq) + } + fmt.Printf("Claude parser extracted %d tool sequences with %d total tool calls\n", + len(metrics.ToolSequences), totalTools) + } + // Convert tool call map to slice for _, toolInfo := range toolCallMap { metrics.ToolCalls = append(metrics.ToolCalls, *toolInfo) @@ -834,13 +844,13 @@ func (e *ClaudeEngine) parseToolCallsWithSequence(contentArray []interface{}, to // Extract tool name if toolName, exists := contentMap["name"]; exists { if nameStr, ok := toolName.(string); ok { - // Skip internal tools as per existing JavaScript logic - internalTools := []string{ - "Read", "Write", "Edit", "MultiEdit", "LS", "Grep", "Glob", "TodoWrite", - } - if slices.Contains(internalTools, nameStr) { - continue - } + // Skip internal tools as per existing JavaScript logic (disabled for tool graph visualization) + // internalTools := []string{ + // "Read", "Write", "Edit", "MultiEdit", "LS", "Grep", "Glob", "TodoWrite", + // } + // if slices.Contains(internalTools, nameStr) { + // continue + // } // Prettify tool name prettifiedName := PrettifyToolName(nameStr) From 470342668dd23cd56e0be5ed469c394fbc6aac1f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:41:20 +0000 Subject: [PATCH 7/7] Fix missing parseToolCalls method causing test failures in ClaudeEngine Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../test-claude-create-pull-request.lock.yml | 2 +- .../test-claude-push-to-branch.lock.yml | 2 +- .../test-codex-create-pull-request.lock.yml | 2 +- .../test-codex-push-to-branch.lock.yml | 2 +- .../test-safe-outputs-custom-engine.lock.yml | 2 +- pkg/workflow/claude_engine.go | 73 ++++++++++++++++++- pkg/workflow/codex_engine.go | 4 +- 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-claude-create-pull-request.lock.yml b/.github/workflows/test-claude-create-pull-request.lock.yml index 7f415dceee9..20fc3b9a3d0 100644 --- a/.github/workflows/test-claude-create-pull-request.lock.yml +++ b/.github/workflows/test-claude-create-pull-request.lock.yml @@ -1541,7 +1541,7 @@ jobs: echo '## Git Patch' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - head -50 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY + head -500 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY echo '...' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-claude-push-to-branch.lock.yml b/.github/workflows/test-claude-push-to-branch.lock.yml index a8510ef91f5..841806a3cf9 100644 --- a/.github/workflows/test-claude-push-to-branch.lock.yml +++ b/.github/workflows/test-claude-push-to-branch.lock.yml @@ -1660,7 +1660,7 @@ jobs: echo '## Git Patch' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - head -50 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY + head -500 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY echo '...' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-codex-create-pull-request.lock.yml b/.github/workflows/test-codex-create-pull-request.lock.yml index cdfb0bc1d50..d13f998a0ea 100644 --- a/.github/workflows/test-codex-create-pull-request.lock.yml +++ b/.github/workflows/test-codex-create-pull-request.lock.yml @@ -1290,7 +1290,7 @@ jobs: echo '## Git Patch' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - head -50 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY + head -500 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY echo '...' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-codex-push-to-branch.lock.yml b/.github/workflows/test-codex-push-to-branch.lock.yml index 18883a8990c..8857c7708e1 100644 --- a/.github/workflows/test-codex-push-to-branch.lock.yml +++ b/.github/workflows/test-codex-push-to-branch.lock.yml @@ -1411,7 +1411,7 @@ jobs: echo '## Git Patch' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - head -50 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY + head -500 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY echo '...' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-safe-outputs-custom-engine.lock.yml b/.github/workflows/test-safe-outputs-custom-engine.lock.yml index 4f01406b60d..487945fa3b9 100644 --- a/.github/workflows/test-safe-outputs-custom-engine.lock.yml +++ b/.github/workflows/test-safe-outputs-custom-engine.lock.yml @@ -1270,7 +1270,7 @@ jobs: echo '## Git Patch' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - head -50 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY + head -500 /tmp/aw.patch >> $GITHUB_STEP_SUMMARY || echo "Could not display patch contents" >> $GITHUB_STEP_SUMMARY echo '...' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo '' >> $GITHUB_STEP_SUMMARY diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index a292a9110dc..e0adb4cf4f9 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -814,9 +814,9 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe currentSequence = append(currentSequence, sequenceInMessage...) } } - } - } - } + } + } + } } } @@ -943,6 +943,73 @@ func (e *ClaudeEngine) parseToolCallsWithSequence(contentArray []interface{}, to return sequence } +// parseToolCalls extracts tool call information from Claude log content array without sequence tracking +func (e *ClaudeEngine) parseToolCalls(contentArray []interface{}, toolCallMap map[string]*ToolCallInfo) { + for _, contentItem := range contentArray { + if contentMap, ok := contentItem.(map[string]interface{}); ok { + if contentType, exists := contentMap["type"]; exists { + if typeStr, ok := contentType.(string); ok && typeStr == "tool_use" { + // Extract tool name + if toolName, exists := contentMap["name"]; exists { + if nameStr, ok := toolName.(string); ok { + // Prettify tool name + prettifiedName := PrettifyToolName(nameStr) + + // Special handling for bash - each invocation is unique + if nameStr == "Bash" { + if input, exists := contentMap["input"]; exists { + if inputMap, ok := input.(map[string]interface{}); ok { + if command, exists := inputMap["command"]; exists { + if commandStr, ok := command.(string); ok { + // Create unique bash entry with command info, avoiding colons + uniqueBashName := fmt.Sprintf("bash_%s", e.shortenCommand(commandStr)) + prettifiedName = uniqueBashName + } + } + } + } + } + + // Initialize or update tool call info + if toolInfo, exists := toolCallMap[prettifiedName]; exists { + toolInfo.CallCount++ + } else { + toolCallMap[prettifiedName] = &ToolCallInfo{ + Name: prettifiedName, + CallCount: 1, + MaxOutputSize: 0, // Will be updated when we find tool results + MaxDuration: 0, // Will be updated when we find execution timing + } + } + } + } + } else if typeStr == "tool_result" { + // Extract output size for tool results + if content, exists := contentMap["content"]; exists { + if contentStr, ok := content.(string); ok { + // Estimate token count (rough approximation: 1 token = ~4 characters) + outputSize := len(contentStr) / 4 + + // Find corresponding tool call to update max output size + if toolUseID, exists := contentMap["tool_use_id"]; exists { + if _, ok := toolUseID.(string); ok { + // This is simplified - in a full implementation we'd track tool_use_id to tool name mapping + // For now, we'll update the max output size for all tools (conservative estimate) + for _, toolInfo := range toolCallMap { + if outputSize > toolInfo.MaxOutputSize { + toolInfo.MaxOutputSize = outputSize + } + } + } + } + } + } + } + } + } + } +} + // shortenCommand creates a short identifier for bash commands func (e *ClaudeEngine) shortenCommand(command string) string { // Take first 20 characters and remove newlines diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 29755ab440e..830f9eba92a 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -313,8 +313,8 @@ func (e *CodexEngine) parseCodexToolCallsWithSequence(line string, toolCallMap m } } } - - return "" // No tool call found + + return "" // No tool call found } // updateMostRecentToolWithDuration updates the tool with maximum duration