From 3231266e49770b7d1cfac694cab34eda31cb6151 Mon Sep 17 00:00:00 2001 From: chupei Date: Wed, 29 Jul 2026 11:14:09 +0800 Subject: [PATCH 1/2] feat: add token usage --- dingo/exec/local.py | 16 ++ dingo/io/output/eval_detail.py | 19 ++- dingo/io/output/result_info.py | 17 +- dingo/io/output/summary_model.py | 59 +++++++ dingo/model/llm/agent/agent_hallucination.py | 3 +- dingo/model/llm/agent/base_agent.py | 3 +- .../model/llm/agent/tools/claims_extractor.py | 25 ++- dingo/model/llm/base.py | 16 +- dingo/model/llm/base_litellm.py | 10 +- dingo/model/llm/base_openai.py | 129 ++++++++++++++- dingo/model/llm/llm_custom_metric.py | 20 ++- dingo/model/llm/llm_factcheck_public.py | 39 ++++- .../llm/llm_search_result_effectiveness.py | 18 +- .../model/llm/llm_search_result_relevance.py | 16 +- .../model/llm/rag/llm_rag_answer_relevancy.py | 20 ++- .../llm/rag/llm_rag_context_precision.py | 11 +- dingo/model/llm/vlm_layout_quality.py | 10 +- docs/config.md | 49 ++++++ test/scripts/exec/test_local.py | 83 +++++++++- test/scripts/io/test_summary_model.py | 44 +++++ test/scripts/model/llm/test_litellm.py | 24 ++- .../model/llm/test_llm_custom_metric.py | 38 +++++ test/scripts/model/llm/test_token_usage.py | 156 ++++++++++++++++++ 23 files changed, 784 insertions(+), 41 deletions(-) create mode 100644 test/scripts/model/llm/test_token_usage.py diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 18f40328..7321015b 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -134,6 +134,13 @@ def execute(self) -> SummaryModel: self.summary.type_count[field_key].setdefault(label, 0) self.summary.type_count[field_key][label] += 1 + for field_key, eval_detail_list in result_info.token_usage_details.items(): + for eval_detail in eval_detail_list: + if eval_detail.usage is not None and eval_detail.metric: + self.summary.add_token_usage( + field_key, eval_detail.metric, eval_detail.usage + ) + if result_info.eval_status: self.summary.num_bad += 1 else: @@ -210,6 +217,9 @@ def evaluate_single_data(self, dingo_id: str, eval_fields: dict, eval_type: str, # Set result_info fields join_fields = ','.join(eval_fields.values()) if eval_fields else 'default' + usage_detail_list = [mr for mr in eval_detail_list if mr.usage is not None] + if usage_detail_list: + result_info.token_usage_details = {join_fields: usage_detail_list} # 根据配置决定保存哪些结果 if self.input_args.executor.result_save.all_labels or self.input_args.executor.result_save.merge: @@ -242,6 +252,12 @@ def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInf # 第一层是字段名，如果不存在，则直接赋值 else: existing_item.eval_details[key] = value + + for key, value in new_item.token_usage_details.items(): + if key in existing_item.token_usage_details: + existing_item.token_usage_details[key].extend(value) + else: + existing_item.token_usage_details[key] = value else: existing_list.append(new_item) diff --git a/dingo/io/output/eval_detail.py b/dingo/io/output/eval_detail.py index f2073dca..03592d35 100644 --- a/dingo/io/output/eval_detail.py +++ b/dingo/io/output/eval_detail.py @@ -1,6 +1,6 @@ -from typing import Any, Dict, List, Optional +from typing import Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel class QualityLabel: @@ -9,6 +9,20 @@ class QualityLabel: QUALITY_BAD_PREFIX = "QUALITY_BAD_" # Indicates not pass the quality check +class TokenUsage(BaseModel): + """Token usage returned by an LLM provider for one evaluator call.""" + + prompt_tokens: Optional[int] = None + completion_tokens: Optional[int] = None + total_tokens: Optional[int] = None + reasoning_tokens: Optional[int] = None + cached_tokens: Optional[int] = None + model: Optional[str] = None + provider: Optional[str] = None + calls: int = 1 + source: str = "provider" + + class EvalDetail(BaseModel): metric: str status: bool = False @@ -16,3 +30,4 @@ class EvalDetail(BaseModel): score: Optional[float] = None label: Optional[list[str]] = None reason: Optional[list] = None + usage: Optional[TokenUsage] = None diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py index 6a4a9549..6adc4abf 100644 --- a/dingo/io/output/result_info.py +++ b/dingo/io/output/result_info.py @@ -5,7 +5,7 @@ from decimal import Decimal from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field from dingo.io.output.eval_detail import EvalDetail @@ -15,6 +15,17 @@ class ResultInfo(BaseModel): raw_data: Dict = {} eval_status: bool = False eval_details: Dict[str, List[EvalDetail]] = {} + token_usage_details: Dict[str, List[EvalDetail]] = Field( + default_factory=dict, + exclude=True, + ) + + @staticmethod + def _eval_detail_to_dict(model_res: EvalDetail) -> Dict[str, Any]: + detail = model_res.model_dump() + if detail.get('usage') is None: + detail.pop('usage', None) + return detail @staticmethod def _apply_field_filter(output_data: Dict[str, Any], field_list: Optional[List[str]]) -> Dict[str, Any]: @@ -82,7 +93,7 @@ def to_dict(self, field_list: Optional[List[str]] = None): 'raw_data': self._normalize_value(self.raw_data), 'eval_status': self.eval_status, 'eval_details': { - k: [model_res.model_dump() for model_res in v] + k: [self._eval_detail_to_dict(model_res) for model_res in v] for k, v in self.eval_details.items() }, } @@ -112,7 +123,7 @@ def move_conflict_field(field_name: str): dingo_result = { 'eval_status': self.eval_status, 'eval_details': { - k: [model_res.model_dump() for model_res in v] + k: [self._eval_detail_to_dict(model_res) for model_res in v] for k, v in self.eval_details.items() }, } diff --git a/dingo/io/output/summary_model.py b/dingo/io/output/summary_model.py index f2de1df2..cb2a2fe9 100644 --- a/dingo/io/output/summary_model.py +++ b/dingo/io/output/summary_model.py @@ -3,6 +3,8 @@ from pydantic import BaseModel, Field +from dingo.io.output.eval_detail import TokenUsage + class SummaryModel(BaseModel): task_id: str = '' @@ -22,6 +24,7 @@ class SummaryModel(BaseModel): # 新增：指标分数统计（用于RAG等评估场景） # 结构：{field_key: {metric_name: {scores, score_average, ...}}} metrics_score_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict) + token_usage_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict) def add_metric_score(self, field_key: str, metric_name: str, score: float): """ @@ -46,6 +49,59 @@ def add_metric_score(self, field_key: str, metric_name: str, score: float): metric_stats['scores'].append(score) metric_stats['score_count'] += 1 + def add_token_usage(self, field_key: str, metric_name: str, usage: TokenUsage): + """ + 添加 LLM token 使用量到统计中 + + Args: + field_key: 字段名（如 'user_input,response'） + metric_name: 指标名称（如 LLMTextQualityV5） + usage: 单次 LLM 调用的 token 使用量 + """ + usage_stats = self.token_usage_stats.setdefault(field_key, {}).setdefault( + metric_name, + { + 'prompt_tokens': 0, + 'completion_tokens': 0, + 'total_tokens': 0, + 'reasoning_tokens': 0, + 'cached_tokens': 0, + 'calls': 0, + 'records': 0, + 'models': {}, + 'providers': {}, + 'sources': {}, + }, + ) + + for token_field in [ + 'prompt_tokens', + 'completion_tokens', + 'total_tokens', + 'reasoning_tokens', + 'cached_tokens', + ]: + value = getattr(usage, token_field, None) + if value is not None: + usage_stats[token_field] += int(value) + + calls = int(usage.calls or 1) + usage_stats['calls'] += calls + usage_stats['records'] += 1 + + if usage.model: + usage_stats['models'][usage.model] = ( + usage_stats['models'].get(usage.model, 0) + calls + ) + if usage.provider: + usage_stats['providers'][usage.provider] = ( + usage_stats['providers'].get(usage.provider, 0) + calls + ) + if usage.source: + usage_stats['sources'][usage.source] = ( + usage_stats['sources'].get(usage.source, 0) + calls + ) + def calculate_metrics_score_averages(self): """ 计算所有字段和指标分数的平均值、最小值、最大值、标准差 @@ -133,4 +189,7 @@ def to_dict(self): for field_key, metrics in self.metrics_score_stats.items() } + if self.token_usage_stats: + result['token_usage'] = self.token_usage_stats + return result diff --git a/dingo/model/llm/agent/agent_hallucination.py b/dingo/model/llm/agent/agent_hallucination.py index fc22ba56..889cd07d 100644 --- a/dingo/model/llm/agent/agent_hallucination.py +++ b/dingo/model/llm/agent/agent_hallucination.py @@ -20,6 +20,7 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model import Model from dingo.model.llm.agent.base_agent import BaseAgent +from dingo.model.llm.base import llm_response_content from dingo.utils import log @@ -339,7 +340,7 @@ def _extract_claims(cls, input_data: Data) -> List[str]: # Call LLM messages = [{"role": "user", "content": prompt}] - response = cls.send_messages(messages) + response = llm_response_content(cls.send_messages(messages)) # Parse JSON response # Handle markdown code blocks diff --git a/dingo/model/llm/agent/base_agent.py b/dingo/model/llm/agent/base_agent.py index d3db23d2..8308ccbd 100644 --- a/dingo/model/llm/agent/base_agent.py +++ b/dingo/model/llm/agent/base_agent.py @@ -16,6 +16,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.llm.agent.tools import ToolRegistry +from dingo.model.llm.base import llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log @@ -251,7 +252,7 @@ def eval(cls, input_data: Data) -> EvalDetail: prompt = step.get('prompt', '') # Use parent's send_messages method messages = [{"role": "user", "content": prompt}] - response = cls.send_messages(messages) + response = llm_response_content(cls.send_messages(messages)) results.append(response) else: diff --git a/dingo/model/llm/agent/tools/claims_extractor.py b/dingo/model/llm/agent/tools/claims_extractor.py index f3204b96..d8dc786c 100644 --- a/dingo/model/llm/agent/tools/claims_extractor.py +++ b/dingo/model/llm/agent/tools/claims_extractor.py @@ -23,8 +23,10 @@ from pydantic import Field +from dingo.io.output.eval_detail import TokenUsage from dingo.model.llm.agent.tools.base_tool import BaseTool, ToolConfig from dingo.model.llm.agent.tools.tool_registry import tool_register +from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log @@ -351,10 +353,11 @@ def execute( # Extract claims from each chunk all_claims = [] + token_usage: TokenUsage | None = None for i, chunk_data in enumerate(chunks): - log.debug(f"Processing chunk {i+1}/{len(chunks)}") + log.debug(f"Processing chunk {i + 1}/{len(chunks)}") - chunk_claims = cls._extract_claims_from_chunk( + chunk_claims, chunk_usage = cls._extract_claims_from_chunk( client, chunk_data['text'], chunk_data['start_pos'], @@ -362,6 +365,7 @@ def execute( include_context ) all_claims.extend(chunk_claims) + token_usage = BaseOpenAI._merge_token_usage(token_usage, chunk_usage) # Deduplicate and merge similar claims unique_claims = cls._deduplicate_claims(all_claims) @@ -377,6 +381,8 @@ def execute( # Build metadata metadata = cls._build_metadata(unique_claims) + if token_usage is not None: + metadata['token_usage'] = token_usage.model_dump() result = { 'success': True, @@ -460,7 +466,7 @@ def _extract_claims_from_chunk( start_pos: int, claim_types: List[str], include_context: bool - ) -> List[Dict]: + ) -> tuple[List[Dict], TokenUsage | None]: """ Extract claims from a single text chunk using LLM. @@ -472,7 +478,7 @@ def _extract_claims_from_chunk( include_context: Whether to include context Returns: - List of extracted claims + Tuple of extracted claims and token usage """ # Build user prompt user_prompt = f"""Extract verifiable claims from the following text. @@ -496,6 +502,11 @@ def _extract_claims_from_chunk( temperature=cls.config.temperature, response_format={"type": "json_object"} # Force JSON output ) + token_usage = BaseOpenAI._extract_token_usage( + response, + model_name=cls.config.model, + provider="openai", + ) output_text = response.choices[0].message.content @@ -520,14 +531,14 @@ def _extract_claims_from_chunk( filtered_claims.append(claim) - return filtered_claims + return filtered_claims, token_usage except json.JSONDecodeError as e: log.warning(f"Failed to parse LLM output as JSON: {e}") - return [] + return [], token_usage if 'token_usage' in locals() else None except Exception as e: log.error(f"LLM call failed: {e}") - return [] + return [], None @classmethod def _deduplicate_claims(cls, claims: List[Dict]) -> List[Dict]: diff --git a/dingo/model/llm/base.py b/dingo/model/llm/base.py index 440193e2..e0fb53cd 100644 --- a/dingo/model/llm/base.py +++ b/dingo/model/llm/base.py @@ -1,8 +1,20 @@ -from typing import List +from typing import List, Optional from dingo.config.input_args import EvaluatorLLMArgs from dingo.io import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage + + +class LLMCallResult: + def __init__(self, content: str, usage: Optional[TokenUsage] = None): + self.content = content + self.usage = usage + + +def llm_response_content(response) -> str: + if isinstance(response, LLMCallResult): + return response.content + return str(response) class BaseLLM: diff --git a/dingo/model/llm/base_litellm.py b/dingo/model/llm/base_litellm.py index bcf92528..b3ffbe93 100644 --- a/dingo/model/llm/base_litellm.py +++ b/dingo/model/llm/base_litellm.py @@ -1,6 +1,7 @@ from typing import List from dingo.config.input_args import EvaluatorLLMArgs +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils.exception import ExceedMaxTokens @@ -106,4 +107,11 @@ def send_messages(cls, messages: List) -> str: ) content = choice.message.content # type: ignore[union-attr] - return str(content) if content is not None else "" + return LLMCallResult( + content=str(content) if content is not None else "", + usage=cls._extract_token_usage( + response, + model_name=model_name, + provider="litellm", + ), + ) diff --git a/dingo/model/llm/base_openai.py b/dingo/model/llm/base_openai.py index c3911699..a644fa3a 100644 --- a/dingo/model/llm/base_openai.py +++ b/dingo/model/llm/base_openai.py @@ -6,8 +6,8 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data, RequiredField -from dingo.io.output.eval_detail import EvalDetail, QualityLabel -from dingo.model.llm.base import BaseLLM +from dingo.io.output.eval_detail import EvalDetail, QualityLabel, TokenUsage +from dingo.model.llm.base import BaseLLM, LLMCallResult from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens @@ -96,7 +96,121 @@ def send_messages(cls, messages: List): f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}" ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=cls._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) + + @staticmethod + def _usage_value(data, key: str): + if data is None: + return None + if isinstance(data, dict): + return data.get(key) + return getattr(data, key, None) + + @staticmethod + def _coerce_optional_int(value): + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + @classmethod + def _extract_token_usage( + cls, + completion, + model_name: str, + provider: str = "openai", + ) -> TokenUsage | None: + raw_usage = getattr(completion, "usage", None) + if raw_usage is None: + return None + + if hasattr(raw_usage, "model_dump"): + usage_data = raw_usage.model_dump() + elif isinstance(raw_usage, dict): + usage_data = raw_usage + else: + usage_data = raw_usage + + completion_details = cls._usage_value( + usage_data, "completion_tokens_details" + ) + prompt_details = cls._usage_value(usage_data, "prompt_tokens_details") + + return TokenUsage( + prompt_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "prompt_tokens") + ), + completion_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "completion_tokens") + ), + total_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "total_tokens") + ), + reasoning_tokens=cls._coerce_optional_int( + cls._usage_value(completion_details, "reasoning_tokens") + ), + cached_tokens=cls._coerce_optional_int( + cls._usage_value(prompt_details, "cached_tokens") + ), + model=model_name, + provider=provider, + source="provider", + ) + + @staticmethod + def _copy_token_usage(usage: TokenUsage) -> TokenUsage: + if hasattr(usage, "model_copy"): + return usage.model_copy(deep=True) + return usage.copy(deep=True) + + @classmethod + def _merge_token_usage( + cls, + current: TokenUsage | None, + new_usage: TokenUsage | None, + ) -> TokenUsage | None: + if new_usage is None: + return current + if current is None: + return cls._copy_token_usage(new_usage) + + def _sum_optional(left, right): + if left is None and right is None: + return None + return int(left or 0) + int(right or 0) + + current.prompt_tokens = _sum_optional( + current.prompt_tokens, new_usage.prompt_tokens + ) + current.completion_tokens = _sum_optional( + current.completion_tokens, new_usage.completion_tokens + ) + current.total_tokens = _sum_optional( + current.total_tokens, new_usage.total_tokens + ) + current.reasoning_tokens = _sum_optional( + current.reasoning_tokens, new_usage.reasoning_tokens + ) + current.cached_tokens = _sum_optional( + current.cached_tokens, new_usage.cached_tokens + ) + current.calls += int(new_usage.calls or 1) + if current.model != new_usage.model: + current.model = current.model or new_usage.model + if current.provider != new_usage.provider: + current.provider = current.provider or new_usage.provider + if current.source != new_usage.source: + current.source = current.source or new_usage.source + return current @classmethod def validate_numeric_range(cls, value, min_val, max_val, param_name): @@ -191,10 +305,16 @@ def eval(cls, input_data: Data) -> EvalDetail: attempts = 0 except_msg = "" except_name = Exception.__class__.__name__ + usage: TokenUsage | None = None while attempts < 3: try: response = cls.send_messages(messages) - res: EvalDetail = cls.process_response(response) + if isinstance(response, LLMCallResult): + usage = cls._merge_token_usage(usage, response.usage) + res: EvalDetail = cls.process_response(response.content) + res.usage = usage + else: + res: EvalDetail = cls.process_response(response) return res except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: except_msg = str(e) @@ -216,4 +336,5 @@ def eval(cls, input_data: Data) -> EvalDetail: res.status = True res.label = [f"QUALITY_BAD.{except_name}"] res.reason = [except_msg] + res.usage = usage return res diff --git a/dingo/model/llm/llm_custom_metric.py b/dingo/model/llm/llm_custom_metric.py index 53d2968e..8624932b 100644 --- a/dingo/model/llm/llm_custom_metric.py +++ b/dingo/model/llm/llm_custom_metric.py @@ -7,6 +7,7 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data from dingo.io.output.eval_detail import EvalDetail +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.model.model import Model from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens @@ -112,7 +113,14 @@ def send_messages(self, messages: List): f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}" ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=self._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) def _eval_detail_from_response(self, response_json: dict) -> EvalDetail: custom_metric = self._get_custom_metric() @@ -189,9 +197,15 @@ def eval(self, input_data: Data) -> EvalDetail: attempts = 0 except_msg = "" except_name = Exception.__name__ + usage = None while attempts < 3: try: response = self.send_messages(messages) + if isinstance(response, LLMCallResult): + usage = self._merge_token_usage(usage, response.usage) + result = self.process_response(response.content) + result.usage = usage + return result return self.process_response(response) except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: except_msg = str(e) @@ -203,9 +217,11 @@ def eval(self, input_data: Data) -> EvalDetail: except_msg = str(e) except_name = e.__class__.__name__ - return EvalDetail( + result = EvalDetail( metric=self._get_custom_metric().metric, status=True, label=[f"QUALITY_BAD.{except_name}"], reason=[except_msg], ) + result.usage = usage + return result diff --git a/dingo/model/llm/llm_factcheck_public.py b/dingo/model/llm/llm_factcheck_public.py index 966c44a1..18e0aee8 100644 --- a/dingo/model/llm/llm_factcheck_public.py +++ b/dingo/model/llm/llm_factcheck_public.py @@ -4,6 +4,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI @@ -199,17 +200,25 @@ def eval(cls, input_data: Data) -> EvalDetail: cls.create_client() # 1. 提取声明 - claims = cls._extract_claims(input_data.prompt, input_data.content) + usage = None + claims, claim_usage = cls._extract_claims_with_usage( + input_data.prompt, input_data.content + ) + usage = cls._merge_token_usage(usage, claim_usage) if not claims: result = EvalDetail(metric=cls.__name__) result.reason = ["No factual claims found"] + result.usage = usage return result # 2. 分批验证 all_results = [] for i in range(0, len(claims), cls.batch_size): batch = claims[i:i + cls.batch_size] - results = cls._verify_claims(input_data.prompt, input_data.content, batch) + results, batch_usage = cls._verify_claims_with_usage( + input_data.prompt, input_data.content, batch + ) + usage = cls._merge_token_usage(usage, batch_usage) all_results.extend(results) # 3. 计算指标 @@ -218,6 +227,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 4. 设置评估结果 result = EvalDetail(metric=cls.__name__) result.reason = [cls._format_reason(metrics)] + result.usage = usage # 5. 根据分数设置状态 if metrics["factual_ratio"] < cls.threshold: @@ -237,6 +247,11 @@ def eval(cls, input_data: Data) -> EvalDetail: @classmethod def _extract_claims(cls, prompt: str, response: str) -> List[str]: + claims, _ = cls._extract_claims_with_usage(prompt, response) + return claims + + @classmethod + def _extract_claims_with_usage(cls, prompt: str, response: str): """提取事实性声明""" messages = [ {"role": "user", "content": (cls.prompt["CLAIM_LISTING"] + @@ -245,10 +260,12 @@ def _extract_claims(cls, prompt: str, response: str) -> List[str]: response=response )} ] - result = cls.send_messages(messages) + response_result = cls.send_messages(messages) + result = llm_response_content(response_result) try: claims = cls._parse_json_list(result) - return [c for c in claims if c.strip()] # 过滤空声明 + usage = response_result.usage if isinstance(response_result, LLMCallResult) else None + return [c for c in claims if c.strip()], usage # 过滤空声明 except Exception as e: raise ValueError(f"Failed to parse claims: {str(e)}") @@ -257,6 +274,14 @@ def _verify_claims(cls, prompt: str, response: str, claims: List[str]) -> List[FactCheckResult]: + results, _ = cls._verify_claims_with_usage(prompt, response, claims) + return results + + @classmethod + def _verify_claims_with_usage(cls, + prompt: str, + response: str, + claims: List[str]): """验证一批声明""" messages = [ {"role": "user", "content": (cls.prompt["FACT_CHECKING"] + @@ -266,9 +291,11 @@ def _verify_claims(cls, claims=claims )} ] - result = cls.send_messages(messages) + response_result = cls.send_messages(messages) + result = llm_response_content(response_result) try: - return cls._parse_check_results(result) + usage = response_result.usage if isinstance(response_result, LLMCallResult) else None + return cls._parse_check_results(result), usage except Exception as e: raise ValueError(f"Failed to parse check results: {str(e)}") diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index 86e76ae2..fcf25f2c 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -20,8 +20,9 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI logger = logging.getLogger(__name__) @@ -367,6 +368,7 @@ class LLMFieldQuality: issues: list[str] | None = None reason: str = "" error: str = "" + usage: TokenUsage | None = None def field_score(self, field: str) -> float: return { @@ -430,6 +432,7 @@ class EffectivenessGrade: issues: list[str] | None = None llm_quality_reason: str = "" llm_quality_error: str = "" + usage: TokenUsage | None = None def to_dict(self) -> dict[str, Any]: return { @@ -555,6 +558,7 @@ def _judge_llm_field_quality( return LLMFieldQuality() client = self._get_client() last_result = LLMFieldQuality(error="LLM field quality judgment failed") + usage: TokenUsage | None = None for attempt in range(3): try: completion = client.chat.completions.create( @@ -577,14 +581,24 @@ def _judge_llm_field_quality( max_tokens=self.max_tokens, timeout=self.timeout, ) + usage = BaseOpenAI._merge_token_usage( + usage, + BaseOpenAI._extract_token_usage( + completion, + model_name=self.model, + provider="openai", + ), + ) response_text = completion.choices[0].message.content or "" last_result = _parse_llm_field_quality_response(response_text) + last_result.usage = usage if not last_result.error: return last_result error: Exception | str = last_result.error except Exception as exc: error = exc last_result = LLMFieldQuality(error=str(exc)) + last_result.usage = usage logger.warning( "LLM field quality attempt %s/3 failed for title=%r: %s", @@ -722,6 +736,7 @@ def apply_confirmed_field_issue(field: str, score: float) -> float: issues=issues, llm_quality_reason=llm_quality.reason, llm_quality_error=llm_quality.error, + usage=llm_quality.usage, ) @classmethod @@ -766,6 +781,7 @@ def eval(cls, input_data: Data) -> EvalDetail: score=round(grade.score, 5), label=labels, reason=[grade.to_dict()], + usage=grade.usage, ) diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index febea86f..a15c2d0a 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -25,8 +25,9 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI logger = logging.getLogger(__name__) @@ -127,6 +128,7 @@ class RelevanceGrade: confidence: float = 0.0 reasoning: str = "" error: str = "" + usage: TokenUsage | None = None def to_dict(self) -> dict[str, Any]: d: dict[str, Any] = { @@ -475,6 +477,7 @@ def grade( client = self._get_client() last_grade = RelevanceGrade(error="LLM grading failed") + usage: TokenUsage | None = None for attempt in range(3): try: completion = client.chat.completions.create( @@ -487,14 +490,24 @@ def grade( max_tokens=self.max_tokens, timeout=self.timeout, ) + usage = BaseOpenAI._merge_token_usage( + usage, + BaseOpenAI._extract_token_usage( + completion, + model_name=self.model, + provider="openai", + ), + ) response_text = completion.choices[0].message.content or "" last_grade = _parse_grade_response(response_text) + last_grade.usage = usage if not last_grade.error: return last_grade error: Exception | str = last_grade.error except Exception as exc: error = exc last_grade = RelevanceGrade(error=str(exc)) + last_grade.usage = usage logger.warning( "LLM grading attempt %s/3 failed for query=%r title=%r: %s", @@ -577,6 +590,7 @@ def eval(cls, input_data: Data) -> EvalDetail: score=round(grade.score, 5), label=labels, reason=[reason], + usage=grade.usage, ) diff --git a/dingo/model/llm/rag/llm_rag_answer_relevancy.py b/dingo/model/llm/rag/llm_rag_answer_relevancy.py index 86febfa7..898d31b2 100644 --- a/dingo/model/llm/rag/llm_rag_answer_relevancy.py +++ b/dingo/model/llm/rag/llm_rag_answer_relevancy.py @@ -13,6 +13,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.exception import ConvertJsonError @@ -106,7 +107,14 @@ def build_messages(cls, input_data: Data) -> List: @classmethod def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[str, Any]]: """生成多个相关问题""" + questions, _ = cls._generate_multiple_questions_with_usage(input_data, n) + return questions + + @classmethod + def _generate_multiple_questions_with_usage(cls, input_data: Data, n: int = 3): + """生成多个相关问题，同时返回 LLM token 使用量""" questions = [] + usage = None # 确保客户端已经创建 if not hasattr(cls, 'client') or cls.client is None: @@ -117,13 +125,16 @@ def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[ messages = cls.build_messages(input_data) # 调用LLM生成问题 - response = cls.send_messages(messages) + response_result = cls.send_messages(messages) + response = llm_response_content(response_result) + if isinstance(response_result, LLMCallResult): + usage = cls._merge_token_usage(usage, response_result.usage) # 处理响应 processed_response = cls.process_question_response(response) questions.append(processed_response) - return questions + return questions, usage @classmethod def process_question_response(cls, response: str) -> Dict[str, Any]: @@ -246,7 +257,9 @@ def eval(cls, input_data: Data) -> EvalDetail: cls.dynamic_config.temperature = 0.7 # 生成多个相关问题 - generated_questions = cls.generate_multiple_questions(input_data, cls.strictness) + generated_questions, usage = cls._generate_multiple_questions_with_usage( + input_data, cls.strictness + ) # 计算相关性分数和详细信息 score, details = cls.calculate_score(generated_questions, original_question) @@ -254,6 +267,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 构建结果 result = EvalDetail(metric=cls.__name__) result.score = score + result.usage = usage # 根据分数判断是否通过，默认阈值为5 threshold = 5 diff --git a/dingo/model/llm/rag/llm_rag_context_precision.py b/dingo/model/llm/rag/llm_rag_context_precision.py index a5927ec3..c1be34ed 100644 --- a/dingo/model/llm/rag/llm_rag_context_precision.py +++ b/dingo/model/llm/rag/llm_rag_context_precision.py @@ -10,6 +10,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.exception import ConvertJsonError @@ -293,6 +294,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 获取所有上下文的消息 messages_list = cls.build_messages(input_data) responses = [] + usage = None # 为每个上下文发送单独的请求 for item in messages_list: @@ -302,7 +304,10 @@ def eval(cls, input_data: Data) -> EvalDetail: while attempts < 3: try: - response = cls.send_messages(messages) + response_result = cls.send_messages(messages) + response = llm_response_content(response_result) + if isinstance(response_result, LLMCallResult): + usage = cls._merge_token_usage(usage, response_result.usage) break except Exception as e: attempts += 1 @@ -326,4 +331,6 @@ def eval(cls, input_data: Data) -> EvalDetail: responses.append(response) # 处理所有响应 - return cls.process_response(responses) + result = cls.process_response(responses) + result.usage = usage + return result diff --git a/dingo/model/llm/vlm_layout_quality.py b/dingo/model/llm/vlm_layout_quality.py index 14c03fc2..812ad145 100644 --- a/dingo/model/llm/vlm_layout_quality.py +++ b/dingo/model/llm/vlm_layout_quality.py @@ -4,6 +4,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.image_loader import ImageLoader @@ -159,7 +160,14 @@ def send_messages(cls, messages: List): temperature=0.1 ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=cls._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) @classmethod def process_response(cls, response: str) -> EvalDetail: diff --git a/docs/config.md b/docs/config.md index ec595fcf..ba7ba129 100644 --- a/docs/config.md +++ b/docs/config.md @@ -100,6 +100,55 @@ HuggingFace 特定配置： | all_labels | bool | false | No | 是否保存所有标签 | | raw | bool | false | No | 是否保存原始数据 | +### LLM Token 使用量输出 + +当 LLM 服务返回 token usage 时，Dingo 会在对应的 `EvalDetail` 中写入 `usage` 字段，并在 `summary.json` 中按字段组合和 evaluator 汇总到 `token_usage`。该统计来自模型服务商返回的 `usage`，不会本地估算；如果兼容 API 不返回 usage，则对应字段为空。 + +单条结果示例： + +```json +{ + "metric": "LLMTextQualityV5", + "status": false, + "label": ["QUALITY_GOOD"], + "reason": ["pass"], + "usage": { + "prompt_tokens": 812, + "completion_tokens": 96, + "total_tokens": 908, + "reasoning_tokens": null, + "cached_tokens": null, + "model": "gpt-4o-mini", + "provider": "openai", + "calls": 1, + "source": "provider" + } +} +``` + +汇总结果示例： + +```json +{ + "token_usage": { + "content": { + "LLMTextQualityV5": { + "prompt_tokens": 81200, + "completion_tokens": 9600, + "total_tokens": 90800, + "reasoning_tokens": 0, + "cached_tokens": 12000, + "calls": 100, + "records": 100, + "models": {"gpt-4o-mini": 100}, + "providers": {"openai": 100}, + "sources": {"provider": 100} + } + } + } +} +``` + ### Evaluator 配置 (evaluator) 评估器相关配置： diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index 44ca5014..f96f3525 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -3,7 +3,8 @@ from dingo.config import InputArgs from dingo.exec import Executor, LocalExecutor from dingo.io import ResultInfo -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage +from dingo.model import Model class TestLocal: @@ -192,6 +193,86 @@ def test_merge_result_info(self): assert "�I am 8 years old. ^I love apple because:" in all_reasons assert "文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。" in all_reasons + def test_merge_result_info_preserves_token_usage_details(self): + localexecutor = LocalExecutor({}) + item1 = ResultInfo( + dingo_id="1", + token_usage_details={ + "content": [ + EvalDetail( + metric="LLMMetricA", + usage=TokenUsage(total_tokens=3), + ) + ] + }, + ) + item2 = ResultInfo( + dingo_id="1", + token_usage_details={ + "content": [ + EvalDetail( + metric="LLMMetricB", + usage=TokenUsage(total_tokens=5), + ) + ] + }, + ) + + merged = localexecutor.merge_result_info([], item1) + merged = localexecutor.merge_result_info(merged, item2) + + assert len(merged[0].token_usage_details["content"]) == 2 + + def test_token_usage_summary_can_include_unsaved_good_eval_details(self): + class TokenUsageGoodLLM: + def eval(self, input_data): + return EvalDetail( + metric="TokenUsageGoodLLM", + status=False, + label=["QUALITY_GOOD"], + usage=TokenUsage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + ), + ) + + old_model = Model.llm_name_map.get("TokenUsageGoodLLM") + Model.llm_name_map["TokenUsageGoodLLM"] = TokenUsageGoodLLM + try: + input_args = InputArgs( + executor={ + "result_save": { + "bad": True, + "good": False, + "all_labels": False, + } + }, + evaluator=[ + { + "fields": {"content": "content"}, + "evals": [{"name": "TokenUsageGoodLLM"}], + } + ], + ) + executor = LocalExecutor(input_args) + + result = executor.evaluate_single_data( + dingo_id="1", + eval_fields={"content": "content"}, + eval_type="llm", + map_data={"content": "ok"}, + eval_list=input_args.evaluator[0].evals, + ) + + assert result.eval_details == {} + assert result.token_usage_details["content"][0].usage.total_tokens == 15 + finally: + if old_model is None: + Model.llm_name_map.pop("TokenUsageGoodLLM", None) + else: + Model.llm_name_map["TokenUsageGoodLLM"] = old_model + def test_all_labels_config(self): input_data = { "input_path": "test/data/test_local_jsonl.jsonl", diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py index c9420ffc..9e7fac5f 100644 --- a/test/scripts/io/test_summary_model.py +++ b/test/scripts/io/test_summary_model.py @@ -11,6 +11,7 @@ import pytest +from dingo.io.output.eval_detail import TokenUsage from dingo.io.output.summary_model import SummaryModel @@ -221,6 +222,49 @@ def test_to_dict_without_scores(self): # 验证没有分数统计字段 assert "metrics_score" not in result + def test_add_token_usage_and_to_dict(self): + """测试 LLM token 使用量统计输出""" + summary = SummaryModel(task_name="test_task", task_id="test_token_001") + + summary.add_token_usage( + "content", + "LLMTextQualityV5", + TokenUsage( + prompt_tokens=10, + completion_tokens=4, + total_tokens=14, + reasoning_tokens=1, + cached_tokens=3, + model="gpt-test", + provider="openai", + ), + ) + summary.add_token_usage( + "content", + "LLMTextQualityV5", + TokenUsage( + prompt_tokens=8, + completion_tokens=5, + total_tokens=13, + model="gpt-test", + provider="openai", + ), + ) + + result = summary.to_dict() + + stats = result["token_usage"]["content"]["LLMTextQualityV5"] + assert stats["prompt_tokens"] == 18 + assert stats["completion_tokens"] == 9 + assert stats["total_tokens"] == 27 + assert stats["reasoning_tokens"] == 1 + assert stats["cached_tokens"] == 3 + assert stats["calls"] == 2 + assert stats["records"] == 2 + assert stats["models"] == {"gpt-test": 2} + assert stats["providers"] == {"openai": 2} + assert stats["sources"] == {"provider": 2} + def test_multiple_metrics_different_score_counts(self): """测试不同指标有不同数量的分数""" summary = SummaryModel( diff --git a/test/scripts/model/llm/test_litellm.py b/test/scripts/model/llm/test_litellm.py index 43d1ed90..124d5ee4 100644 --- a/test/scripts/model/llm/test_litellm.py +++ b/test/scripts/model/llm/test_litellm.py @@ -26,12 +26,12 @@ class _Provider(BaseLiteLLM): return _Provider -def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop"): +def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop", usage=None): choice = SimpleNamespace( finish_reason=finish_reason, message=SimpleNamespace(content=content), ) - return SimpleNamespace(choices=[choice]) + return SimpleNamespace(choices=[choice], usage=usage) # --------------------------------------------------------------------------- @@ -127,7 +127,25 @@ def test_none_content_returns_empty_string(self): none_resp = _stub_response(content=None) with mock.patch("litellm.completion", return_value=none_resp): result = P.send_messages([{"role": "user", "content": "hi"}]) - assert result == "" + assert result.content == "" + + def test_returns_token_usage_when_provider_supplies_usage(self): + P = _make_provider(model="gpt-4o") + provider_resp = _stub_response( + usage={ + "prompt_tokens": 6, + "completion_tokens": 3, + "total_tokens": 9, + } + ) + with mock.patch("litellm.completion", return_value=provider_resp): + result = P.send_messages([{"role": "user", "content": "hi"}]) + + assert result.content == '{"score": 1, "reason": "ok"}' + assert result.usage.prompt_tokens == 6 + assert result.usage.completion_tokens == 3 + assert result.usage.total_tokens == 9 + assert result.usage.provider == "litellm" # --------------------------------------------------------------------------- diff --git a/test/scripts/model/llm/test_llm_custom_metric.py b/test/scripts/model/llm/test_llm_custom_metric.py index 5eeefdf7..5665bf3d 100644 --- a/test/scripts/model/llm/test_llm_custom_metric.py +++ b/test/scripts/model/llm/test_llm_custom_metric.py @@ -3,6 +3,8 @@ from dingo.config.input_args import EvaluatorLLMArgs, InputArgs from dingo.io.input import Data +from dingo.io.output.eval_detail import TokenUsage +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.llm_custom_metric import LLMCustomMetric from dingo.model.model import Model @@ -197,6 +199,42 @@ def test_eval_detail_response_uses_llm_returned_fields(): assert result.reason == ["The content contains AI-style phrasing."] +def test_eval_detail_response_attaches_token_usage(): + llm = LLMCustomMetric() + Model.set_config_llm( + llm, EvaluatorLLMArgs(custom_metric=_custom_metric(metric="SourceLabel")) + ) + llm.create_client = Mock() + llm.send_messages = Mock( + return_value=LLMCallResult( + content=json.dumps( + { + "status": False, + "label": ["SOURCE.AI_GENERATED"], + "score": 0.82, + "reason": ["The content contains AI-style phrasing."], + } + ), + usage=TokenUsage( + prompt_tokens=12, + completion_tokens=5, + total_tokens=17, + model="gpt-test", + provider="openai", + ), + ) + ) + + result = llm.eval( + Data(prompt="Classify source", content="As an AI language model...") + ) + + assert result.usage is not None + assert result.usage.prompt_tokens == 12 + assert result.usage.completion_tokens == 5 + assert result.usage.total_tokens == 17 + + def test_eval_detail_response_rejects_missing_fields(): llm = LLMCustomMetric() Model.set_config_llm( diff --git a/test/scripts/model/llm/test_token_usage.py b/test/scripts/model/llm/test_token_usage.py new file mode 100644 index 00000000..aea2779e --- /dev/null +++ b/test/scripts/model/llm/test_token_usage.py @@ -0,0 +1,156 @@ +from types import SimpleNamespace + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io import ResultInfo +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model.llm.base import LLMCallResult +from dingo.model.llm.base_openai import BaseOpenAI + + +def _completion( + content='{"score": 1, "reason": "ok"}', + usage=None, + finish_reason="stop", +): + return SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason=finish_reason, + message=SimpleNamespace(content=content), + ) + ], + usage=usage, + ) + + +def test_extract_token_usage_from_openai_response_object(): + usage = SimpleNamespace( + prompt_tokens=11, + completion_tokens=7, + total_tokens=18, + prompt_tokens_details=SimpleNamespace(cached_tokens=3), + completion_tokens_details=SimpleNamespace(reasoning_tokens=2), + ) + + result = BaseOpenAI._extract_token_usage( + _completion(usage=usage), + model_name="gpt-test", + provider="openai", + ) + + assert result.prompt_tokens == 11 + assert result.completion_tokens == 7 + assert result.total_tokens == 18 + assert result.cached_tokens == 3 + assert result.reasoning_tokens == 2 + assert result.model == "gpt-test" + assert result.provider == "openai" + assert result.calls == 1 + + +def test_base_openai_eval_attaches_token_usage(): + class UsageLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return LLMCallResult( + content='{"score": 1, "reason": "ok"}', + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 5, + "completion_tokens": 2, + "total_tokens": 7, + } + ), + model_name="gpt-test", + ), + ) + + result = UsageLLM.eval(Data(content="sample")) + + assert result.status is False + assert result.usage is not None + assert result.usage.prompt_tokens == 5 + assert result.usage.completion_tokens == 2 + assert result.usage.total_tokens == 7 + + +def test_base_openai_error_result_keeps_token_usage(): + class ParseErrorLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return LLMCallResult( + content="not json", + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 3, + "completion_tokens": 1, + "total_tokens": 4, + } + ), + model_name="gpt-test", + ), + ) + + result = ParseErrorLLM.eval(Data(content="sample")) + + assert result.status is True + assert result.label == ["QUALITY_BAD.ConvertJsonError"] + assert result.usage is not None + assert result.usage.total_tokens == 4 + + +def test_base_openai_eval_still_accepts_legacy_string_send_messages(): + class LegacyLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return '{"score": 1, "reason": "ok"}' + + result = LegacyLLM.eval(Data(content="sample")) + + assert result.status is False + assert result.usage is None + + +def test_result_info_only_serializes_usage_when_present(): + with_usage = ResultInfo( + dingo_id="1", + eval_details={ + "content": [ + EvalDetail( + metric="LLMMetric", + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 1, + "completion_tokens": 2, + "total_tokens": 3, + } + ), + model_name="gpt-test", + ), + ) + ] + }, + ).to_dict() + without_usage = ResultInfo( + dingo_id="2", + eval_details={"content": [EvalDetail(metric="RuleMetric")]}, + ).to_dict() + + assert with_usage["eval_details"]["content"][0]["usage"]["total_tokens"] == 3 + assert "usage" not in without_usage["eval_details"]["content"][0] From abf2222621ceae2923ce2db3bc3389348be31344 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 29 Jul 2026 03:15:42 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9A=20Auto-update=20metrics=20docu?= =?UTF-8?q?mentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 156 ++++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 72 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index da990a9c..f339d264 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,11 +8,11 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | ### Pretrain Text Quality Assessment Metrics @@ -25,10 +25,10 @@ This document provides comprehensive information about all quality metrics used | `LLMMathCompare` | LLMMathCompare | Compares the effectiveness of two tools in extracting mathematical formulas from HTML to Markdown format by evaluatin... | Internal Implementation | N/A | N/A | | `LLMSecurityPolitics` | LLMSecurityPolitics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | N/A | | `LLMTableCompare` | LLMTableCompare | Compares the effectiveness of two tools in extracting tables from HTML to Markdown format by evaluating recognition r... | Internal Implementation | N/A | N/A | -| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | -| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | +| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | ### SFT Data Assessment Metrics @@ -36,82 +36,37 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `LLMFactCheckPublic` | LLMFactCheckPublic | Two-stage factuality evaluation pipeline from GPT-5 | [GPT-5 System Card](https://cdn.openai.com/pdf/8124a3ce-ab78-4f06-96eb-49ea29ffb52f/gpt5-system-card-aug7.pdf) (OpenAI) | N/A | N/A | | `LLMHallucination` | LLMHallucination | Evaluates whether the response contains factual contradictions or hallucinations against provided context information | [TruthfulQA: Measuring How Models Mimic Human Falsehoods](https://arxiv.org/abs/2109.07958) (Lin et al., 2021) | N/A | N/A | -| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [See Results](Returns clarity score (0-10) and detailed analysis) | [View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [See Results](Returns difficulty level (1-10) with detailed breakdown) | [View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [📊 See Results](Returns clarity score (0-10) and detailed analysis) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [📊 See Results](Returns difficulty level (1-10) with detailed breakdown) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A | ### Classification Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | +| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [📊 See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | ### Multimodality Assessment Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `LLMClassifyQR` | LLMClassifyQR | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | N/A | -| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用 DeepSeek-OCR 作为 Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [See Results](通过对比 VLM 输出与 OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | - -### TC609-5-2025-04 Quality Evaluation Metrics - -| Type | Rule | Coverage | Group | Description | -|---|---|---|---|---| -| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | -| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | -| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | -| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | partial | `guobiao_data` | Combines existing NLP, SFT, image, audio, and video format rules. | -| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | partial | `guobiao_data` | Combines unsafe-word, PII, and identity-card detection. | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | partial | `guobiao_data` | Combines image-label overlap and visualization checks. | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | partial | `guobiao_data` | Combines null-content and short-content checks. | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | partial | `guobiao_data` | Uses HHEM consistency checking as partial evidence of authenticity. | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | partial | `guobiao_data` | Combines structured-field and image-text consistency checks. | -| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | partial | `guobiao_data` | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | -| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | covered | `pretrain,guobiao_text` | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | -| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | partial | `guobiao_text` | Combines alphabetic-word, stop-word, and unique-word ratio checks. | -| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | covered | `guobiao_text` | Combines document-text and formula repetition checks. | -| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | covered | `guobiao_text` | Combines null, short, ellipsis-ending, and terminal-ending checks. | -| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | partial | `guobiao_text` | Uses content length and sentence/word counts as partial missing-information checks. | -| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | partial | `guobiao_text` | Combines abnormal HTML, character, invisible-content, and watermark checks. | -| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | partial | `guobiao_text` | Combines punctuation, word-boundary, and line-break fluency checks. | -| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | partial | `guobiao_image` | Uses image aspect-ratio validation as partial resolution coverage. | -| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | covered | `guobiao_image` | Uses PHash and CNN duplicate-image detection. | -| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | partial | `guobiao_image` | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | -| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | partial | `guobiao_image` | Combines image validity and NIMA quality as partial clarity coverage. | -| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | uncovered | `guobiao_video` | Placeholder: video resolution is not implemented. | -| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | uncovered | `guobiao_video` | Placeholder: duplicate-video detection is not implemented. | -| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | uncovered | `guobiao_video` | Placeholder: video FPS validation is not implemented. | -| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | uncovered | `guobiao_video` | Placeholder: video duration validation is not implemented. | -| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | uncovered | `guobiao_video` | Placeholder: video clarity evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | uncovered | `guobiao_video` | Placeholder: video dynamic-range evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | covered | `guobiao_audio` | Uses the existing Welch power-spectrum SNR implementation. | -| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | uncovered | `guobiao_audio` | Placeholder: signal distortion ratio is not implemented. | -| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | uncovered | `guobiao_audio` | Placeholder: sample-rate quality validation is not implemented. | -| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | uncovered | `guobiao_audio` | Placeholder: audio bit-depth validation is not implemented. | -| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | uncovered | `guobiao_audio` | Placeholder: audio bit-rate validation is not implemented. | -| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | covered | `guobiao_audio` | Uses the existing WAV duration implementation. | -| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | partial | `guobiao_data` | Combines available text cleanliness checks; modality coverage is partial. | -| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | uncovered | `guobiao_model` | Placeholder: target-scenario distribution coverage is not implemented. | -| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | uncovered | `guobiao_model` | Placeholder: dataset scale versus model requirements is not implemented. | -| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | covered | `guobiao_model` | Checks whether created and updated timestamps are within configured time ranges | -| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | partial | `guobiao_model` | Uses image annotation checks as partial evidence of annotation accuracy. | -| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | uncovered | `guobiao_model` | Placeholder: before/after model performance comparison is not implemented. | +| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用DeepSeek-OCR作为Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [📊 See Results](通过对比VLM输出与OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | ### Rule-Based TEXT Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; Checks PDF content for abnormal ch... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics @@ -131,6 +86,12 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_EFFECTIVENESS` | RuleAudioDuration | Check whether the audio duration meets the standard | Internal Implementation | N/A | N/A | | `QUALITY_BAD_EFFECTIVENESS` | RuleAudioSnrQuality | Check whether the audio signal-to-noise ratio meets the standard | Internal Implementation | N/A | N/A | +### Document Quality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `LLMAISmell` | LLMAISmell | Detects AI-generated writing patterns in requirement documents across 5 dimensions: hollow truisms, repetition, rainb... | Internal Implementation | N/A | [📝 View Example](../examples/llm_and_rule/llm_local.py) | + ### Job Hunting Strategy Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | @@ -146,18 +107,30 @@ This document provides comprehensive information about all quality metrics used | `LLMMetaRaterReadability` | LLMMetaRaterReadability | Evaluates the clarity and coherence of text using appropriate vocabulary and sentence structures on a 5-point scale | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | | `LLMMetaRaterReasoning` | LLMMetaRaterReasoning | Evaluates the reasoning complexity and logical depth of text content, from simple logical judgments to complex multid... | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | +### National Standard Data Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and s... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | + ### OCR Eval Metric | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | -| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | +| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | +| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | ### RAG Retrieved Evidence Chunk Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [View Example](../examples/rag/sdk_chunk_eval.py) | +| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [📝 View Example](../examples/rag/sdk_chunk_eval.py) | ### Resume Quality Assessment Metrics @@ -171,7 +144,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为 0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics @@ -185,6 +158,44 @@ This document provides comprehensive information about all quality metrics used | `RESUME_QUALITY_BAD_PROFESSIONALISM` | RuleResumeEmoji, RuleResumeInformal | Detects emoji usage in resume which reduces professionalism; Detects informal or colloquial expressions in resume | Internal Implementation | N/A | N/A | | `RESUME_QUALITY_BAD_STRUCTURE` | RuleResumeNameMissing, RuleResumeSectionMissing | Checks if resume contains a name in the first 200 characters; Checks if resume contains required sections like educat... | Internal Implementation | N/A | N/A | +### SAC/TC609 High-quality Dataset Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Combines existing NLP, SFT, image, audio, and video format rules. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Combines image-label overlap and visualization checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Uses HHEM consistency checking as partial evidence of authenticity. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | Uses content length and sentence/word counts as partial missing-information checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | Combines abnormal HTML, character, invisible-content, and watermark checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | Combines punctuation, word-boundary, and line-break fluency checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | Uses image aspect-ratio validation as partial resolution coverage. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | Uses PHash and CNN duplicate-image detection. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | Combines image validity and NIMA quality as partial clarity coverage. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | Placeholder: video resolution is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | Placeholder: duplicate-video detection is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | Placeholder: video FPS validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | Placeholder: video duration validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | Placeholder: video clarity evaluation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | Placeholder: video dynamic-range evaluation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | Uses the existing Welch power-spectrum SNR implementation. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | Placeholder: signal distortion ratio is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | Placeholder: sample-rate quality validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | Placeholder: audio bit-depth validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | Placeholder: audio bit-rate validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | Uses the existing WAV duration implementation. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | Combines available text cleanliness checks; modality coverage is partial. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | Placeholder: target-scenario distribution coverage is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | Placeholder: dataset scale versus model requirements is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | Uses image annotation checks as partial evidence of annotation accuracy. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | Placeholder: before/after model performance comparison is not implemented. | Internal Implementation | N/A | N/A | + ### SFT Data Assessment Metrics - Agent-Enhanced | Type | Metric | Description | Paper Source | Evaluation Results | Examples | @@ -204,3 +215,4 @@ This document provides comprehensive information about all quality metrics used | `AgentFactCheck` | AgentFactCheck | Agent-based hallucination detection with autonomous web search | Internal Implementation | N/A | N/A | | `ArticleFactChecker` | ArticleFactChecker | Article-level fact checking with autonomous claims extraction and verification | Internal Implementation | N/A | N/A | | `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | +