feat: 8 missing eval datasets, sources for 12 external skills, 100% skillmgr coverage

cmd/sin-code/internal/agentloop/budget.go

@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: MIT
+// Purpose: Budget enforcement — token and cost limits per session and
+// project (issue #320). The Budget struct accumulates token and USD
+// spend and exposes level/remaining/percent queries so the agent loop
+// can stop before overspending. Thread-safe (mandate M7).
+package agentloop
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// BudgetLevel classifies how much of the budget has been consumed.
+type BudgetLevel int
+
+const (
+	// BudgetGreen means usage is below 60%.
+	BudgetGreen BudgetLevel = iota
+	// BudgetYellow means usage is between 60% and 90% (inclusive).
+	BudgetYellow
+	// BudgetRed means usage exceeds 90%.
+	BudgetRed
+)
+
+func (l BudgetLevel) String() string {
+	switch l {
+	case BudgetGreen:
+		return "green"
+	case BudgetYellow:
+		return "yellow"
+	case BudgetRed:
+		return "red"
+	default:
+		return "unknown"
+	}
+}
+
+// ErrBudgetExhausted is returned by Consume when either the token or cost
+// limit has been exceeded.
+var ErrBudgetExhausted = errors.New("agentloop: budget exhausted")
+
+// Budget enforces token and cost limits for a session or project. A zero
+// limit (maxTokens == 0 or maxCostUSD == 0) means unlimited for that
+// dimension — only non-zero limits are enforced.
+//
+// All methods are safe for concurrent use (mandate M7).
+type Budget struct {
+	mu         sync.Mutex
+	maxTokens  int
+	maxCostUSD float64
+	usedTokens int
+	usedCost   float64
+}
+
+// NewBudget creates a Budget with the given limits. Zero means unlimited
+// for that dimension.
+func NewBudget(maxTokens int, maxCostUSD float64) *Budget {
+	return &Budget{
+		maxTokens:  maxTokens,
+		maxCostUSD: maxCostUSD,
+	}
+}
+
+// Consume records token and cost usage. It always adds the usage to the
+// running totals (so tracking remains accurate even when over budget) and
+// then returns ErrBudgetExhausted if either non-zero limit is exceeded.
+func (b *Budget) Consume(tokens int, cost float64) error {
+	if b == nil {
+		return nil
+	}
+	if tokens < 0 {
+		tokens = 0
+	}
+	if cost < 0 {
+		cost = 0
+	}
+	b.mu.Lock()
+	b.usedTokens += tokens
+	b.usedCost += cost
+	exceeded := false
+	var detail string
+	if b.maxTokens > 0 && b.usedTokens > b.maxTokens {
+		exceeded = true
+		detail = fmt.Sprintf("tokens %d > %d", b.usedTokens, b.maxTokens)
+	}
+	if b.maxCostUSD > 0 && b.usedCost > b.maxCostUSD {
+		exceeded = true
+		if detail != "" {
+			detail += "; "
+		}
+		detail += fmt.Sprintf("cost $%.4f > $%.4f", b.usedCost, b.maxCostUSD)
+	}
+	b.mu.Unlock()
+	if exceeded {
+		return fmt.Errorf("%w: %s", ErrBudgetExhausted, detail)
+	}
+	return nil
+}
+
+// Remaining returns the remaining token and cost budget. For unlimited
+// dimensions (limit == 0) the remaining value is -1, signalling
+// "unlimited" to the caller.
+func (b *Budget) Remaining() (int, float64) {
+	if b == nil {
+		return -1, -1
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	remTokens := -1
+	if b.maxTokens > 0 {
+		remTokens = b.maxTokens - b.usedTokens
+		if remTokens < 0 {
+			remTokens = 0
+		}
+	}
+	remCost := -1.0
+	if b.maxCostUSD > 0 {
+		remCost = b.maxCostUSD - b.usedCost
+		if remCost < 0 {
+			remCost = 0
+		}
+	}
+	return remTokens, remCost
+}
+
+// Percent returns the fraction of the budget that has been consumed,
+// expressed as a value between 0.0 and 1.0+. When both limits are zero
+// (unlimited), it returns 0. When only one dimension has a limit, that
+// dimension's percentage is used. When both have limits, the higher of
+// the two is returned — the caller should act on the most-consumed
+// dimension.
+func (b *Budget) Percent() float64 {
+	if b == nil {
+		return 0
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var tokPct, costPct float64
+	if b.maxTokens > 0 {
+		tokPct = float64(b.usedTokens) / float64(b.maxTokens)
+	}
+	if b.maxCostUSD > 0 {
+		costPct = b.usedCost / b.maxCostUSD
+	}
+	if tokPct >= costPct {
+		return tokPct
+	}
+	return costPct
+}
+
+// IsExhausted reports whether either non-zero limit has been exceeded.
+func (b *Budget) IsExhausted() bool {
+	if b == nil {
+		return false
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.maxTokens > 0 && b.usedTokens > b.maxTokens {
+		return true
+	}
+	if b.maxCostUSD > 0 && b.usedCost > b.maxCostUSD {
+		return true
+	}
+	return false
+}
+
+// Level returns the budget level based on Percent(): Green < 60%,
+// Yellow 60–90%, Red > 90%.
+func (b *Budget) Level() BudgetLevel {
+	if b == nil {
+		return BudgetGreen
+	}
+	pct := b.Percent()
+	switch {
+	case pct > 0.9:
+		return BudgetRed
+	case pct >= 0.6:
+		return BudgetYellow
+	default:
+		return BudgetGreen
+	}
+}
+
+// Reset zeroes the accumulated usage. Limits are preserved.
+func (b *Budget) Reset() {
+	if b == nil {
+		return
+	}
+	b.mu.Lock()
+	b.usedTokens = 0
+	b.usedCost = 0
+	b.mu.Unlock()
+}
+
+// UsedTokens returns the total tokens consumed so far.
+func (b *Budget) UsedTokens() int {
+	if b == nil {
+		return 0
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.usedTokens
+}
+
+// UsedCost returns the total USD cost consumed so far.
+func (b *Budget) UsedCost() float64 {
+	if b == nil {
+		return 0
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.usedCost
+}
+
+// MaxTokens returns the configured token limit (0 = unlimited).
+func (b *Budget) MaxTokens() int {
+	if b == nil {
+		return 0
+	}
+	return b.maxTokens
+}
+
+// MaxCostUSD returns the configured cost limit (0 = unlimited).
+func (b *Budget) MaxCostUSD() float64 {
+	if b == nil {
+		return 0
+	}
+	return b.maxCostUSD
+}

cmd/sin-code/internal/agentloop/budget_test.go

@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: MIT
+// Purpose: unit tests for the Budget enforcer (issue #320, M7).
+package agentloop
+
+import (
+	"errors"
+	"sync"
+	"testing"
+)
+
+func TestBudget_Consume_UnderLimit_NoError(t *testing.T) {
+	b := NewBudget(10000, 5.0)
+	if err := b.Consume(3000, 1.0); err != nil {
+		t.Fatalf("expected no error under limit, got: %v", err)
+	}
+	remTok, remCost := b.Remaining()
+	if remTok != 7000 {
+		t.Errorf("remaining tokens: got %d, want 7000", remTok)
+	}
+	if remCost != 4.0 {
+		t.Errorf("remaining cost: got %.2f, want 4.0", remCost)
+	}
+}
+
+func TestBudget_Consume_OverTokenLimit_Error(t *testing.T) {
+	b := NewBudget(1000, 5.0)
+	if err := b.Consume(1500, 0.5); !errors.Is(err, ErrBudgetExhausted) {
+		t.Fatalf("expected ErrBudgetExhausted, got: %v", err)
+	}
+	if !b.IsExhausted() {
+		t.Error("expected IsExhausted=true after exceeding token limit")
+	}
+}
+
+func TestBudget_Consume_OverCostLimit_Error(t *testing.T) {
+	b := NewBudget(10000, 0.10)
+	if err := b.Consume(100, 0.20); !errors.Is(err, ErrBudgetExhausted) {
+		t.Fatalf("expected ErrBudgetExhausted, got: %v", err)
+	}
+	if !b.IsExhausted() {
+		t.Error("expected IsExhausted=true after exceeding cost limit")
+	}
+}
+
+func TestBudget_Unlimited_NoError(t *testing.T) {
+	b := NewBudget(0, 0)
+	if err := b.Consume(9_999_999, 9_999.0); err != nil {
+		t.Fatalf("unlimited budget should never error, got: %v", err)
+	}
+	if b.IsExhausted() {
+		t.Error("unlimited budget should not be exhausted")
+	}
+	remTok, remCost := b.Remaining()
+	if remTok != -1 || remCost != -1 {
+		t.Errorf("unlimited remaining should be -1, -1; got %d, %.2f", remTok, remCost)
+	}
+}
+
+func TestBudget_Percent(t *testing.T) {
+	b := NewBudget(1000, 10.0)
+	b.Consume(300, 0)
+	if pct := b.Percent(); pct < 0.29 || pct > 0.31 {
+		t.Errorf("percent after 300/1000 tokens: got %.2f, want ~0.30", pct)
+	}
+	b.Consume(0, 5.0)
+	if pct := b.Percent(); pct < 0.49 || pct > 0.51 {
+		t.Errorf("percent after $5/$10 cost: got %.2f, want ~0.50", pct)
+	}
+}
+
+func TestBudget_Level(t *testing.T) {
+	b := NewBudget(1000, 10.0)
+	b.Consume(200, 0)
+	if b.Level() != BudgetGreen {
+		t.Errorf("at 20%%: got %s, want green", b.Level())
+	}
+	b.Reset()
+	b.Consume(650, 0)
+	if b.Level() != BudgetYellow {
+		t.Errorf("at 65%%: got %s, want yellow", b.Level())
+	}
+	b.Reset()
+	b.Consume(950, 0)
+	if b.Level() != BudgetRed {
+		t.Errorf("at 95%%: got %s, want red", b.Level())
+	}
+}
+
+func TestBudget_Reset(t *testing.T) {
+	b := NewBudget(1000, 5.0)
+	b.Consume(500, 2.5)
+	b.Reset()
+	if b.IsExhausted() {
+		t.Error("after reset, budget should not be exhausted")
+	}
+	if tok := b.UsedTokens(); tok != 0 {
+		t.Errorf("after reset, used tokens: got %d, want 0", tok)
+	}
+	if cost := b.UsedCost(); cost != 0 {
+		t.Errorf("after reset, used cost: got %.2f, want 0", cost)
+	}
+	if lvl := b.Level(); lvl != BudgetGreen {
+		t.Errorf("after reset, level: got %s, want green", lvl)
+	}
+}
+
+func TestBudget_Remaining_OverLimit_Zero(t *testing.T) {
+	b := NewBudget(1000, 5.0)
+	b.Consume(2000, 10.0)
+	remTok, remCost := b.Remaining()
+	if remTok != 0 {
+		t.Errorf("remaining tokens after overshoot: got %d, want 0", remTok)
+	}
+	if remCost != 0 {
+		t.Errorf("remaining cost after overshoot: got %.2f, want 0", remCost)
+	}
+}
+
+func TestBudget_NilSafe(t *testing.T) {
+	var b *Budget
+	if err := b.Consume(100, 1.0); err != nil {
+		t.Errorf("nil Consume should be no-op, got: %v", err)
+	}
+	if b.IsExhausted() {
+		t.Error("nil IsExhausted should be false")
+	}
+	if b.Level() != BudgetGreen {
+		t.Error("nil Level should be green")
+	}
+	if pct := b.Percent(); pct != 0 {
+		t.Errorf("nil Percent should be 0, got %.2f", pct)
+	}
+	tok, cost := b.Remaining()
+	if tok != -1 || cost != -1 {
+		t.Errorf("nil Remaining should be -1, -1; got %d, %.2f", tok, cost)
+	}
+	b.Reset()
+}
+
+func TestBudget_RaceSafe(t *testing.T) {
+	b := NewBudget(100000, 100.0)
+	var wg sync.WaitGroup
+	for i := 0; i < 200; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_ = b.Consume(100, 0.1)
+			_ = b.IsExhausted()
+			_ = b.Level()
+			_, _ = b.Remaining()
+			_ = b.Percent()
+		}()
+	}
+	wg.Wait()
+	tok := b.UsedTokens()
+	if tok != 20000 {
+		t.Errorf("after 200×100 consume, used tokens: got %d, want 20000", tok)
+	}
+}