diff --git a/cmd/cli/createMsg.go b/cmd/cli/createMsg.go index e9c257e..3de6a81 100644 --- a/cmd/cli/createMsg.go +++ b/cmd/cli/createMsg.go @@ -354,25 +354,74 @@ func generateMessage(ctx context.Context, provider llm.Provider, changes string, // generateMessageWithCache generates a commit message with caching support. func generateMessageWithCache(ctx context.Context, provider llm.Provider, store *store.StoreMethods, providerType types.LLMProvider, changes string, opts *types.GenerationOptions) (string, error) { + startTime := time.Now() + + // Determine if this is a first attempt (cache check eligible) + isFirstAttempt := opts == nil || opts.Attempt <= 1 + // Check cache first (only for first attempt to avoid caching regenerations) - if opts == nil || opts.Attempt <= 1 { + if isFirstAttempt { if cachedEntry, found := store.GetCachedMessage(providerType, changes, opts); found { pterm.Info.Printf("Using cached commit message (saved $%.4f)\n", cachedEntry.Cost) + + // Record cache hit event + event := &types.GenerationEvent{ + Provider: providerType, + Success: true, + GenerationTime: float64(time.Since(startTime).Nanoseconds()) / 1e6, // Convert to milliseconds + TokensUsed: 0, // No tokens used for cached result + Cost: 0, // No cost for cached result + CacheHit: true, + CacheChecked: true, + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + + if err := store.RecordGenerationEvent(event); err != nil { + // Log the error but don't fail the operation + fmt.Printf("Warning: Failed to record usage statistics: %v\n", err) + } + return cachedEntry.Message, nil } } // Generate new message message, err := provider.Generate(ctx, changes, opts) + generationTime := float64(time.Since(startTime).Nanoseconds()) / 1e6 // Convert to milliseconds + + // Estimate tokens and cost + inputTokens := estimateTokens(types.BuildCommitPrompt(changes, opts)) + outputTokens := 100 // Estimate output tokens + cost := estimateCost(providerType, inputTokens, outputTokens) + + // Record generation event + event := &types.GenerationEvent{ + Provider: providerType, + Success: err == nil, + GenerationTime: generationTime, + TokensUsed: inputTokens + outputTokens, + Cost: cost, + CacheHit: false, + CacheChecked: isFirstAttempt, // Only first attempts check cache + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + + if err != nil { + event.ErrorMessage = err.Error() + } + + // Record the event regardless of success/failure + if statsErr := store.RecordGenerationEvent(event); statsErr != nil { + // Log the error but don't fail the operation + fmt.Printf("Warning: Failed to record usage statistics: %v\n", statsErr) + } + if err != nil { return "", err } // Cache the result (only for first attempt) - if opts == nil || opts.Attempt <= 1 { - // Estimate cost for caching - cost := estimateCost(providerType, estimateTokens(types.BuildCommitPrompt(changes, opts)), 100) - + if isFirstAttempt { // Store in cache if cacheErr := store.SetCachedMessage(providerType, changes, opts, message, cost, nil); cacheErr != nil { // Log cache error but don't fail the generation diff --git a/cmd/cli/root.go b/cmd/cli/root.go index 6e417b6..43a5d2b 100644 --- a/cmd/cli/root.go +++ b/cmd/cli/root.go @@ -141,6 +141,7 @@ func init() { rootCmd.AddCommand(creatCommitMsg) rootCmd.AddCommand(llmCmd) rootCmd.AddCommand(cacheCmd) + rootCmd.AddCommand(statsCmd) llmCmd.AddCommand(llmSetupCmd) llmCmd.AddCommand(llmUpdateCmd) cacheCmd.AddCommand(cacheStatsCmd) diff --git a/cmd/cli/stats.go b/cmd/cli/stats.go new file mode 100644 index 0000000..9b802df --- /dev/null +++ b/cmd/cli/stats.go @@ -0,0 +1,195 @@ +package cmd + +import ( + "fmt" + "sort" + "time" + + "github.com/dfanso/commit-msg/cmd/cli/store" + "github.com/dfanso/commit-msg/pkg/types" + "github.com/pterm/pterm" + "github.com/spf13/cobra" +) + +// statsCmd represents the statistics command +var statsCmd = &cobra.Command{ + Use: "stats", + Short: "Display usage statistics", + Long: `Display comprehensive usage statistics including: +- Most used LLM provider +- Average generation time +- Success/failure rates +- Token usage per provider +- Cache hit rates +- Cost tracking`, + RunE: func(cmd *cobra.Command, args []string) error { + Store, err := store.NewStoreMethods() + if err != nil { + return fmt.Errorf("failed to initialize store: %w", err) + } + + reset, _ := cmd.Flags().GetBool("reset") + if reset { + if err := resetStatistics(Store); err != nil { + return err + } + return nil + } + + detailed, _ := cmd.Flags().GetBool("detailed") + return displayStatistics(Store, detailed) + }, +} + +func init() { + statsCmd.Flags().Bool("detailed", false, "Show detailed per-provider statistics") + statsCmd.Flags().Bool("reset", false, "Reset all usage statistics") +} + +func displayStatistics(store *store.StoreMethods, detailed bool) error { + stats := store.GetUsageStats() + + if stats.TotalGenerations == 0 { + pterm.Info.Println("No usage statistics available yet.") + pterm.Info.Println("Statistics will be collected as you use the commit message generator.") + return nil + } + + // Header + pterm.DefaultHeader.WithFullWidth(). + WithBackgroundStyle(pterm.NewStyle(pterm.BgBlue)). + WithTextStyle(pterm.NewStyle(pterm.FgWhite, pterm.Bold)). + Println("Usage Statistics") + + pterm.Println() + + // Overall Statistics + pterm.DefaultSection.WithLevel(2).Println("Overall Statistics") + + overallData := [][]string{ + {"Total Generations", fmt.Sprintf("%d", stats.TotalGenerations)}, + {"Successful Generations", fmt.Sprintf("%d (%.1f%%)", stats.SuccessfulGenerations, store.GetOverallSuccessRate())}, + {"Failed Generations", fmt.Sprintf("%d (%.1f%%)", stats.FailedGenerations, float64(stats.FailedGenerations)/float64(stats.TotalGenerations)*100)}, + {"Average Generation Time", fmt.Sprintf("%.1f ms", stats.AverageGenerationTime)}, + {"Total Cost", fmt.Sprintf("$%.4f", stats.TotalCost)}, + {"Total Tokens Used", fmt.Sprintf("%d", stats.TotalTokensUsed)}, + } + + if stats.CacheHits > 0 || stats.CacheMisses > 0 { + cacheRate := store.GetCacheHitRate() + overallData = append(overallData, []string{"Cache Hit Rate", fmt.Sprintf("%.1f%% (%d hits, %d misses)", cacheRate, stats.CacheHits, stats.CacheMisses)}) + } + + if stats.FirstUse != "" { + if firstUse, err := time.Parse(time.RFC3339, stats.FirstUse); err == nil { + overallData = append(overallData, []string{"First Use", firstUse.Local().Format("Jan 2, 2006 15:04")}) + } + } + + if stats.LastUse != "" { + if lastUse, err := time.Parse(time.RFC3339, stats.LastUse); err == nil { + overallData = append(overallData, []string{"Last Use", lastUse.Local().Format("Jan 2, 2006 15:04")}) + } + } + + pterm.DefaultTable.WithHasHeader(false).WithData(overallData).Render() + pterm.Println() + + // Provider Rankings + if len(stats.ProviderStats) > 0 { + pterm.DefaultSection.WithLevel(2).Println("Provider Rankings") + + ranking := store.GetProviderRanking() + rankingData := [][]string{{"Rank", "Provider", "Uses", "Success Rate", "Avg Time (ms)", "Total Cost"}} + + for i, provider := range ranking { + providerStats := stats.ProviderStats[provider] + rankingData = append(rankingData, []string{ + fmt.Sprintf("#%d", i+1), + string(provider), + fmt.Sprintf("%d", providerStats.TotalUses), + fmt.Sprintf("%.1f%%", providerStats.SuccessRate), + fmt.Sprintf("%.1f", providerStats.AverageGenerationTime), + fmt.Sprintf("$%.4f", providerStats.TotalCost), + }) + } + + pterm.DefaultTable.WithHasHeader(true).WithData(rankingData).Render() + pterm.Println() + } + + // Detailed Provider Statistics + if detailed && len(stats.ProviderStats) > 0 { + pterm.DefaultSection.WithLevel(2).Println("Detailed Provider Statistics") + + // Sort providers alphabetically for consistent display + var providers []types.LLMProvider + for provider := range stats.ProviderStats { + providers = append(providers, provider) + } + sort.Slice(providers, func(i, j int) bool { + return string(providers[i]) < string(providers[j]) + }) + + for _, provider := range providers { + providerStats := stats.ProviderStats[provider] + + pterm.DefaultSection.WithLevel(3).Printf("%s Details", provider) + + providerData := [][]string{ + {"Total Uses", fmt.Sprintf("%d", providerStats.TotalUses)}, + {"Successful Uses", fmt.Sprintf("%d", providerStats.SuccessfulUses)}, + {"Failed Uses", fmt.Sprintf("%d", providerStats.FailedUses)}, + {"Success Rate", fmt.Sprintf("%.1f%%", providerStats.SuccessRate)}, + {"Average Generation Time", fmt.Sprintf("%.1f ms", providerStats.AverageGenerationTime)}, + {"Total Cost", fmt.Sprintf("$%.4f", providerStats.TotalCost)}, + {"Total Tokens Used", fmt.Sprintf("%d", providerStats.TotalTokensUsed)}, + } + + if providerStats.FirstUsed != "" { + if firstUsed, err := time.Parse(time.RFC3339, providerStats.FirstUsed); err == nil { + providerData = append(providerData, []string{"First Used", firstUsed.Local().Format("Jan 2, 2006 15:04")}) + } + } + + if providerStats.LastUsed != "" { + if lastUsed, err := time.Parse(time.RFC3339, providerStats.LastUsed); err == nil { + providerData = append(providerData, []string{"Last Used", lastUsed.Local().Format("Jan 2, 2006 15:04")}) + } + } + + pterm.DefaultTable.WithHasHeader(false).WithData(providerData).Render() + pterm.Println() + } + } + + // Show tips + pterm.DefaultSection.WithLevel(2).Println("Tips") + pterm.Info.Println("• Use --detailed flag to see comprehensive per-provider statistics") + pterm.Info.Println("• Statistics help identify your most reliable and cost-effective providers") + pterm.Info.Println("• Cache hits save both time and API costs") + pterm.Info.Println("• Use --reset flag to clear all statistics (irreversible)") + + return nil +} + +func resetStatistics(store *store.StoreMethods) error { + pterm.Warning.Println("This will permanently delete all usage statistics.") + + confirm, _ := pterm.DefaultInteractiveConfirm. + WithDefaultValue(false). + WithDefaultText("Are you sure you want to reset all statistics?"). + Show() + + if !confirm { + pterm.Info.Println("Statistics reset cancelled.") + return nil + } + + if err := store.ResetUsageStats(); err != nil { + return fmt.Errorf("failed to reset statistics: %w", err) + } + + pterm.Success.Println("All usage statistics have been reset.") + return nil +} \ No newline at end of file diff --git a/cmd/cli/store/store.go b/cmd/cli/store/store.go index b5b14d5..cda2acf 100644 --- a/cmd/cli/store/store.go +++ b/cmd/cli/store/store.go @@ -11,6 +11,7 @@ import ( "github.com/99designs/keyring" "github.com/dfanso/commit-msg/internal/cache" + "github.com/dfanso/commit-msg/internal/usage" "github.com/dfanso/commit-msg/pkg/types" StoreUtils "github.com/dfanso/commit-msg/utils" ) @@ -18,6 +19,7 @@ import ( type StoreMethods struct { ring keyring.Keyring cache *cache.CacheManager + usage *usage.StatsManager } // NewStoreMethods creates a new StoreMethods instance with cache support. @@ -34,9 +36,15 @@ func NewStoreMethods() (*StoreMethods, error) { return nil, fmt.Errorf("failed to initialize cache: %w", err) } + usageManager, err := usage.NewStatsManager() + if err != nil { + return nil, fmt.Errorf("failed to initialize usage statistics: %w", err) + } + return &StoreMethods{ ring: ring, cache: cacheManager, + usage: usageManager, }, nil } @@ -409,3 +417,45 @@ func (s *StoreMethods) GetCacheStats() *types.CacheStats { func (s *StoreMethods) CleanupCache() error { return s.cache.Cleanup() } + +// Usage statistics management methods + +// GetUsageManager returns the usage statistics manager instance. +func (s *StoreMethods) GetUsageManager() *usage.StatsManager { + return s.usage +} + +// RecordGenerationEvent records a commit message generation event for statistics. +func (s *StoreMethods) RecordGenerationEvent(event *types.GenerationEvent) error { + return s.usage.RecordGeneration(event) +} + +// GetUsageStats returns comprehensive usage statistics. +func (s *StoreMethods) GetUsageStats() *types.UsageStats { + return s.usage.GetStats() +} + +// GetMostUsedProvider returns the most frequently used LLM provider. +func (s *StoreMethods) GetMostUsedProvider() (types.LLMProvider, int) { + return s.usage.GetMostUsedProvider() +} + +// GetOverallSuccessRate returns the overall success rate as a percentage. +func (s *StoreMethods) GetOverallSuccessRate() float64 { + return s.usage.GetSuccessRate() +} + +// GetCacheHitRate returns the cache hit rate as a percentage. +func (s *StoreMethods) GetCacheHitRate() float64 { + return s.usage.GetCacheHitRate() +} + +// GetProviderRanking returns providers ranked by usage frequency. +func (s *StoreMethods) GetProviderRanking() []types.LLMProvider { + return s.usage.GetProviderRanking() +} + +// ResetUsageStats clears all usage statistics. +func (s *StoreMethods) ResetUsageStats() error { + return s.usage.ResetStats() +} diff --git a/internal/usage/usage.go b/internal/usage/usage.go new file mode 100644 index 0000000..e27989c --- /dev/null +++ b/internal/usage/usage.go @@ -0,0 +1,276 @@ +package usage + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/dfanso/commit-msg/pkg/types" + StoreUtils "github.com/dfanso/commit-msg/utils" +) + +// StatsManager handles usage statistics tracking and persistence. +type StatsManager struct { + mu sync.RWMutex + stats *types.UsageStats + filePath string +} + +// NewStatsManager creates a new statistics manager instance. +func NewStatsManager() (*StatsManager, error) { + configPath, err := StoreUtils.GetConfigPath() + if err != nil { + return nil, fmt.Errorf("failed to get config path: %w", err) + } + + // Get the directory from the config path + configDir := filepath.Dir(configPath) + statsPath := filepath.Join(configDir, "usage_stats.json") + + manager := &StatsManager{ + filePath: statsPath, + stats: &types.UsageStats{ + ProviderStats: make(map[types.LLMProvider]*types.ProviderStats), + }, + } + + // Load existing stats if they exist + if err := manager.load(); err != nil { + // If file doesn't exist, that's okay - we'll create it on first save + if !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to load existing stats: %w", err) + } + } + + return manager, nil +} + +// RecordGeneration records a commit message generation event. +func (sm *StatsManager) RecordGeneration(event *types.GenerationEvent) error { + sm.mu.Lock() + defer sm.mu.Unlock() + + now := time.Now().UTC().Format(time.RFC3339) + + // Update global stats + sm.stats.TotalGenerations++ + if event.Success { + sm.stats.SuccessfulGenerations++ + } else { + sm.stats.FailedGenerations++ + } + + sm.stats.TotalCost += event.Cost + sm.stats.TotalTokensUsed += event.TokensUsed + sm.stats.LastUse = now + + if sm.stats.FirstUse == "" { + sm.stats.FirstUse = now + } + + // Update cache stats (only when cache was actually checked) + if event.CacheChecked { + if event.CacheHit { + sm.stats.CacheHits++ + } else { + sm.stats.CacheMisses++ + } + } + + // Update average generation time + totalTime := sm.stats.AverageGenerationTime * float64(sm.stats.TotalGenerations-1) + sm.stats.AverageGenerationTime = (totalTime + event.GenerationTime) / float64(sm.stats.TotalGenerations) + + // Update provider-specific stats + if sm.stats.ProviderStats[event.Provider] == nil { + sm.stats.ProviderStats[event.Provider] = &types.ProviderStats{ + Name: event.Provider, + FirstUsed: now, + } + } + + providerStats := sm.stats.ProviderStats[event.Provider] + providerStats.TotalUses++ + if event.Success { + providerStats.SuccessfulUses++ + } else { + providerStats.FailedUses++ + } + + providerStats.TotalCost += event.Cost + providerStats.TotalTokensUsed += event.TokensUsed + providerStats.LastUsed = now + + // Update provider average generation time + totalProviderTime := providerStats.AverageGenerationTime * float64(providerStats.TotalUses-1) + providerStats.AverageGenerationTime = (totalProviderTime + event.GenerationTime) / float64(providerStats.TotalUses) + + // Calculate success rate + if providerStats.TotalUses > 0 { + providerStats.SuccessRate = float64(providerStats.SuccessfulUses) / float64(providerStats.TotalUses) * 100 + } + + // Save to disk + return sm.save() +} + +// GetStats returns a copy of the current usage statistics. +func (sm *StatsManager) GetStats() *types.UsageStats { + sm.mu.RLock() + defer sm.mu.RUnlock() + + // Create a deep copy to prevent external modifications + statsCopy := &types.UsageStats{ + TotalGenerations: sm.stats.TotalGenerations, + SuccessfulGenerations: sm.stats.SuccessfulGenerations, + FailedGenerations: sm.stats.FailedGenerations, + FirstUse: sm.stats.FirstUse, + LastUse: sm.stats.LastUse, + TotalCost: sm.stats.TotalCost, + TotalTokensUsed: sm.stats.TotalTokensUsed, + CacheHits: sm.stats.CacheHits, + CacheMisses: sm.stats.CacheMisses, + AverageGenerationTime: sm.stats.AverageGenerationTime, + ProviderStats: make(map[types.LLMProvider]*types.ProviderStats), + } + + // Deep copy provider stats + for provider, stats := range sm.stats.ProviderStats { + statsCopy.ProviderStats[provider] = &types.ProviderStats{ + Name: stats.Name, + TotalUses: stats.TotalUses, + SuccessfulUses: stats.SuccessfulUses, + FailedUses: stats.FailedUses, + TotalCost: stats.TotalCost, + TotalTokensUsed: stats.TotalTokensUsed, + AverageGenerationTime: stats.AverageGenerationTime, + FirstUsed: stats.FirstUsed, + LastUsed: stats.LastUsed, + SuccessRate: stats.SuccessRate, + } + } + + return statsCopy +} + +// GetMostUsedProvider returns the provider with the highest usage count. +func (sm *StatsManager) GetMostUsedProvider() (types.LLMProvider, int) { + sm.mu.RLock() + defer sm.mu.RUnlock() + + var mostUsed types.LLMProvider + maxUses := 0 + + for provider, stats := range sm.stats.ProviderStats { + if stats.TotalUses > maxUses { + maxUses = stats.TotalUses + mostUsed = provider + } + } + + return mostUsed, maxUses +} + +// GetSuccessRate returns the overall success rate as a percentage. +func (sm *StatsManager) GetSuccessRate() float64 { + sm.mu.RLock() + defer sm.mu.RUnlock() + + if sm.stats.TotalGenerations == 0 { + return 0.0 + } + + return float64(sm.stats.SuccessfulGenerations) / float64(sm.stats.TotalGenerations) * 100 +} + +// GetCacheHitRate returns the cache hit rate as a percentage. +func (sm *StatsManager) GetCacheHitRate() float64 { + sm.mu.RLock() + defer sm.mu.RUnlock() + + totalCacheAttempts := sm.stats.CacheHits + sm.stats.CacheMisses + if totalCacheAttempts == 0 { + return 0.0 + } + + return float64(sm.stats.CacheHits) / float64(totalCacheAttempts) * 100 +} + +// ResetStats clears all usage statistics. +func (sm *StatsManager) ResetStats() error { + sm.mu.Lock() + defer sm.mu.Unlock() + + sm.stats = &types.UsageStats{ + ProviderStats: make(map[types.LLMProvider]*types.ProviderStats), + } + + return sm.save() +} + +// load reads statistics from the persistent storage file. +func (sm *StatsManager) load() error { + data, err := os.ReadFile(sm.filePath) + if err != nil { + return err + } + + if len(data) == 0 { + return nil // Empty file is okay + } + + return json.Unmarshal(data, sm.stats) +} + +// save writes current statistics to the persistent storage file. +func (sm *StatsManager) save() error { + // Ensure directory exists + if err := os.MkdirAll(filepath.Dir(sm.filePath), 0755); err != nil { + return fmt.Errorf("failed to create stats directory: %w", err) + } + + data, err := json.MarshalIndent(sm.stats, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal stats: %w", err) + } + + return os.WriteFile(sm.filePath, data, 0600) +} + +// GetProviderRanking returns providers ranked by usage count. +func (sm *StatsManager) GetProviderRanking() []types.LLMProvider { + sm.mu.RLock() + defer sm.mu.RUnlock() + + type providerUsage struct { + provider types.LLMProvider + uses int + } + + var rankings []providerUsage + for provider, stats := range sm.stats.ProviderStats { + rankings = append(rankings, providerUsage{ + provider: provider, + uses: stats.TotalUses, + }) + } + + // Sort by usage count (descending) + for i := 0; i < len(rankings)-1; i++ { + for j := i + 1; j < len(rankings); j++ { + if rankings[j].uses > rankings[i].uses { + rankings[i], rankings[j] = rankings[j], rankings[i] + } + } + } + + result := make([]types.LLMProvider, len(rankings)) + for i, r := range rankings { + result[i] = r.provider + } + + return result +} \ No newline at end of file diff --git a/pkg/types/types.go b/pkg/types/types.go index 6b1a11a..b665a2b 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -141,3 +141,45 @@ type CacheConfig struct { CleanupInterval int `json:"cleanup_interval_hours"` CacheFilePath string `json:"cache_file_path"` } + +// UsageStats tracks comprehensive usage statistics for the application. +type UsageStats struct { + TotalGenerations int `json:"total_generations"` + SuccessfulGenerations int `json:"successful_generations"` + FailedGenerations int `json:"failed_generations"` + ProviderStats map[LLMProvider]*ProviderStats `json:"provider_stats"` + FirstUse string `json:"first_use"` + LastUse string `json:"last_use"` + TotalCost float64 `json:"total_cost"` + TotalTokensUsed int `json:"total_tokens_used"` + CacheHits int `json:"cache_hits"` + CacheMisses int `json:"cache_misses"` + AverageGenerationTime float64 `json:"average_generation_time_ms"` +} + +// ProviderStats tracks statistics for a specific LLM provider. +type ProviderStats struct { + Name LLMProvider `json:"name"` + TotalUses int `json:"total_uses"` + SuccessfulUses int `json:"successful_uses"` + FailedUses int `json:"failed_uses"` + TotalCost float64 `json:"total_cost"` + TotalTokensUsed int `json:"total_tokens_used"` + AverageGenerationTime float64 `json:"average_generation_time_ms"` + FirstUsed string `json:"first_used"` + LastUsed string `json:"last_used"` + SuccessRate float64 `json:"success_rate"` +} + +// GenerationEvent represents a single commit message generation event for tracking. +type GenerationEvent struct { + Provider LLMProvider `json:"provider"` + Success bool `json:"success"` + GenerationTime float64 `json:"generation_time_ms"` + TokensUsed int `json:"tokens_used"` + Cost float64 `json:"cost"` + CacheHit bool `json:"cache_hit"` + CacheChecked bool `json:"cache_checked"` + Timestamp string `json:"timestamp"` + ErrorMessage string `json:"error_message,omitempty"` +}