feat: add token usage

dingo/exec/local.py

@@ -134,6 +134,13 @@ def execute(self) -> SummaryModel:
                             self.summary.type_count[field_key].setdefault(label, 0)
                             self.summary.type_count[field_key][label] += 1
 
+                    for field_key, eval_detail_list in result_info.token_usage_details.items():
+                        for eval_detail in eval_detail_list:
+                            if eval_detail.usage is not None and eval_detail.metric:
+                                self.summary.add_token_usage(
+                                    field_key, eval_detail.metric, eval_detail.usage
+                                )
+
                     if result_info.eval_status:
                         self.summary.num_bad += 1
                     else:
@@ -210,6 +217,9 @@ def evaluate_single_data(self, dingo_id: str, eval_fields: dict, eval_type: str,
 
         # Set result_info fields
         join_fields = ','.join(eval_fields.values()) if eval_fields else 'default'
+        usage_detail_list = [mr for mr in eval_detail_list if mr.usage is not None]
+        if usage_detail_list:
+            result_info.token_usage_details = {join_fields: usage_detail_list}
 
         # 根据配置决定保存哪些结果
         if self.input_args.executor.result_save.all_labels or self.input_args.executor.result_save.merge:
@@ -242,6 +252,12 @@ def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInf
                 # 第一层是字段名，如果不存在，则直接赋值
                 else:
                     existing_item.eval_details[key] = value
+
+            for key, value in new_item.token_usage_details.items():
+                if key in existing_item.token_usage_details:
+                    existing_item.token_usage_details[key].extend(value)
+                else:
+                    existing_item.token_usage_details[key] = value
         else:
             existing_list.append(new_item)
 

dingo/io/output/eval_detail.py

@@ -1,6 +1,6 @@
-from typing import Any, Dict, List, Optional
+from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 
 class QualityLabel:
@@ -9,10 +9,25 @@ class QualityLabel:
     QUALITY_BAD_PREFIX = "QUALITY_BAD_"  # Indicates not pass the quality check
 
 
+class TokenUsage(BaseModel):
+    """Token usage returned by an LLM provider for one evaluator call."""
+
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    reasoning_tokens: Optional[int] = None
+    cached_tokens: Optional[int] = None
+    model: Optional[str] = None
+    provider: Optional[str] = None
+    calls: int = 1
+    source: str = "provider"
+
+
 class EvalDetail(BaseModel):
     metric: str
     status: bool = False
 
     score: Optional[float] = None
     label: Optional[list[str]] = None
     reason: Optional[list] = None
+    usage: Optional[TokenUsage] = None

dingo/io/output/result_info.py

@@ -5,7 +5,7 @@
 from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from dingo.io.output.eval_detail import EvalDetail
 
@@ -15,6 +15,17 @@ class ResultInfo(BaseModel):
     raw_data: Dict = {}
     eval_status: bool = False
     eval_details: Dict[str, List[EvalDetail]] = {}
+    token_usage_details: Dict[str, List[EvalDetail]] = Field(
+        default_factory=dict,
+        exclude=True,
+    )
+
+    @staticmethod
+    def _eval_detail_to_dict(model_res: EvalDetail) -> Dict[str, Any]:
+        detail = model_res.model_dump()
+        if detail.get('usage') is None:
+            detail.pop('usage', None)
+        return detail
 
     @staticmethod
     def _apply_field_filter(output_data: Dict[str, Any], field_list: Optional[List[str]]) -> Dict[str, Any]:
@@ -82,7 +93,7 @@ def to_dict(self, field_list: Optional[List[str]] = None):
             'raw_data': self._normalize_value(self.raw_data),
             'eval_status': self.eval_status,
             'eval_details': {
-                k: [model_res.model_dump() for model_res in v]
+                k: [self._eval_detail_to_dict(model_res) for model_res in v]
                 for k, v in self.eval_details.items()
             },
         }
@@ -112,7 +123,7 @@ def move_conflict_field(field_name: str):
         dingo_result = {
             'eval_status': self.eval_status,
             'eval_details': {
-                k: [model_res.model_dump() for model_res in v]
+                k: [self._eval_detail_to_dict(model_res) for model_res in v]
                 for k, v in self.eval_details.items()
             },
         }

dingo/io/output/summary_model.py

@@ -3,6 +3,8 @@
 
 from pydantic import BaseModel, Field
 
+from dingo.io.output.eval_detail import TokenUsage
+
 
 class SummaryModel(BaseModel):
     task_id: str = ''
@@ -22,6 +24,7 @@ class SummaryModel(BaseModel):
     # 新增：指标分数统计（用于RAG等评估场景）
     # 结构：{field_key: {metric_name: {scores, score_average, ...}}}
     metrics_score_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict)
+    token_usage_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict)
 
     def add_metric_score(self, field_key: str, metric_name: str, score: float):
         """
@@ -46,6 +49,59 @@ def add_metric_score(self, field_key: str, metric_name: str, score: float):
         metric_stats['scores'].append(score)
         metric_stats['score_count'] += 1
 
+    def add_token_usage(self, field_key: str, metric_name: str, usage: TokenUsage):
+        """
+        添加 LLM token 使用量到统计中
+
+        Args:
+            field_key: 字段名（如 'user_input,response'）
+            metric_name: 指标名称（如 LLMTextQualityV5）
+            usage: 单次 LLM 调用的 token 使用量
+        """
+        usage_stats = self.token_usage_stats.setdefault(field_key, {}).setdefault(
+            metric_name,
+            {
+                'prompt_tokens': 0,
+                'completion_tokens': 0,
+                'total_tokens': 0,
+                'reasoning_tokens': 0,
+                'cached_tokens': 0,
+                'calls': 0,
+                'records': 0,
+                'models': {},
+                'providers': {},
+                'sources': {},
+            },
+        )
+
+        for token_field in [
+            'prompt_tokens',
+            'completion_tokens',
+            'total_tokens',
+            'reasoning_tokens',
+            'cached_tokens',
+        ]:
+            value = getattr(usage, token_field, None)
+            if value is not None:
+                usage_stats[token_field] += int(value)
+
+        calls = int(usage.calls or 1)
+        usage_stats['calls'] += calls
+        usage_stats['records'] += 1
+
+        if usage.model:
+            usage_stats['models'][usage.model] = (
+                usage_stats['models'].get(usage.model, 0) + calls
+            )
+        if usage.provider:
+            usage_stats['providers'][usage.provider] = (
+                usage_stats['providers'].get(usage.provider, 0) + calls
+            )
+        if usage.source:
+            usage_stats['sources'][usage.source] = (
+                usage_stats['sources'].get(usage.source, 0) + calls
+            )
+
     def calculate_metrics_score_averages(self):
         """
         计算所有字段和指标分数的平均值、最小值、最大值、标准差
@@ -133,4 +189,7 @@ def to_dict(self):
                 for field_key, metrics in self.metrics_score_stats.items()
             }
 
+        if self.token_usage_stats:
+            result['token_usage'] = self.token_usage_stats
+
         return result

dingo/model/llm/agent/agent_hallucination.py

@@ -20,6 +20,7 @@
 from dingo.io.output.eval_detail import EvalDetail, QualityLabel
 from dingo.model import Model
 from dingo.model.llm.agent.base_agent import BaseAgent
+from dingo.model.llm.base import llm_response_content
 from dingo.utils import log
 
 
@@ -339,7 +340,7 @@ def _extract_claims(cls, input_data: Data) -> List[str]:
 
             # Call LLM
             messages = [{"role": "user", "content": prompt}]
-            response = cls.send_messages(messages)
+            response = llm_response_content(cls.send_messages(messages))
 
             # Parse JSON response
             # Handle markdown code blocks

dingo/model/llm/agent/base_agent.py

@@ -16,6 +16,7 @@
 from dingo.io.input import Data, RequiredField
 from dingo.io.output.eval_detail import EvalDetail, QualityLabel
 from dingo.model.llm.agent.tools import ToolRegistry
+from dingo.model.llm.base import llm_response_content
 from dingo.model.llm.base_openai import BaseOpenAI
 from dingo.utils import log
 
@@ -251,7 +252,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
                     prompt = step.get('prompt', '')
                     # Use parent's send_messages method
                     messages = [{"role": "user", "content": prompt}]
-                    response = cls.send_messages(messages)
+                    response = llm_response_content(cls.send_messages(messages))
                     results.append(response)
 
                 else:

dingo/model/llm/agent/tools/claims_extractor.py

@@ -23,8 +23,10 @@
 
 from pydantic import Field
 
+from dingo.io.output.eval_detail import TokenUsage
 from dingo.model.llm.agent.tools.base_tool import BaseTool, ToolConfig
 from dingo.model.llm.agent.tools.tool_registry import tool_register
+from dingo.model.llm.base_openai import BaseOpenAI
 from dingo.utils import log
 
 
@@ -351,17 +353,19 @@ def execute(
 
             # Extract claims from each chunk
             all_claims = []
+            token_usage: TokenUsage | None = None
             for i, chunk_data in enumerate(chunks):
-                log.debug(f"Processing chunk {i+1}/{len(chunks)}")
+                log.debug(f"Processing chunk {i + 1}/{len(chunks)}")
 
-                chunk_claims = cls._extract_claims_from_chunk(
+                chunk_claims, chunk_usage = cls._extract_claims_from_chunk(
                     client,
                     chunk_data['text'],
                     chunk_data['start_pos'],
                     claim_types_filter,
                     include_context
                 )
                 all_claims.extend(chunk_claims)
+                token_usage = BaseOpenAI._merge_token_usage(token_usage, chunk_usage)
 
             # Deduplicate and merge similar claims
             unique_claims = cls._deduplicate_claims(all_claims)
@@ -377,6 +381,8 @@ def execute(
 
             # Build metadata
             metadata = cls._build_metadata(unique_claims)
+            if token_usage is not None:
+                metadata['token_usage'] = token_usage.model_dump()
 
             result = {
                 'success': True,
@@ -460,7 +466,7 @@ def _extract_claims_from_chunk(
         start_pos: int,
         claim_types: List[str],
         include_context: bool
-    ) -> List[Dict]:
+    ) -> tuple[List[Dict], TokenUsage | None]:
         """
         Extract claims from a single text chunk using LLM.
 
@@ -472,7 +478,7 @@ def _extract_claims_from_chunk(
             include_context: Whether to include context
 
         Returns:
-            List of extracted claims
+            Tuple of extracted claims and token usage
         """
         # Build user prompt
         user_prompt = f"""Extract verifiable claims from the following text.
@@ -496,6 +502,11 @@ def _extract_claims_from_chunk(
                 temperature=cls.config.temperature,
                 response_format={"type": "json_object"}  # Force JSON output
             )
+            token_usage = BaseOpenAI._extract_token_usage(
+                response,
+                model_name=cls.config.model,
+                provider="openai",
+            )
 
             output_text = response.choices[0].message.content
 
@@ -520,14 +531,14 @@ def _extract_claims_from_chunk(
 
                     filtered_claims.append(claim)
 
-            return filtered_claims
+            return filtered_claims, token_usage
 
         except json.JSONDecodeError as e:
             log.warning(f"Failed to parse LLM output as JSON: {e}")
-            return []
+            return [], token_usage if 'token_usage' in locals() else None
         except Exception as e:
             log.error(f"LLM call failed: {e}")
-            return []
+            return [], None
 
     @classmethod
     def _deduplicate_claims(cls, claims: List[Dict]) -> List[Dict]:

dingo/model/llm/base.py

@@ -1,8 +1,20 @@
-from typing import List
+from typing import List, Optional
 
 from dingo.config.input_args import EvaluatorLLMArgs
 from dingo.io import Data
-from dingo.io.output.eval_detail import EvalDetail
+from dingo.io.output.eval_detail import EvalDetail, TokenUsage
+
+
+class LLMCallResult:
+    def __init__(self, content: str, usage: Optional[TokenUsage] = None):
+        self.content = content
+        self.usage = usage
+
+
+def llm_response_content(response) -> str:
+    if isinstance(response, LLMCallResult):
+        return response.content
+    return str(response)
 
 
 class BaseLLM:

dingo/model/llm/base_litellm.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.model.llm.base import LLMCallResult
 from dingo.model.llm.base_openai import BaseOpenAI
 from dingo.utils.exception import ExceedMaxTokens
 
@@ -106,4 +107,11 @@ def send_messages(cls, messages: List) -> str:
             )
 
         content = choice.message.content  # type: ignore[union-attr]
-        return str(content) if content is not None else ""
+        return LLMCallResult(
+            content=str(content) if content is not None else "",
+            usage=cls._extract_token_usage(
+                response,
+                model_name=model_name,
+                provider="litellm",
+            ),
+        )