Skip to content

feat(ai): Normalize model names for a better cost calculation #103508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 18, 2025
Merged

feat(ai): Normalize model names for a better cost calculation #103508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 23 additions & 62 deletions src/sentry/tasks/ai_agent_monitoring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,55 +26,28 @@
MODELS_DEV_API_URL = "https://models.dev/api.json"


def _create_suffix_glob_model_name(model_id: str) -> str:
def _normalize_model_id(model_id: str) -> str:
"""
Create a suffix glob version of a model name by stripping dates and versions.

Examples:
- "claude-4-sonnet-20250522" -> "claude-4-sonnet-*"
- "o3-pro-2025-06-10" -> "o3-pro-*"
- "claude-3-5-haiku@20241022" -> "claude-3-5-haiku@*"
- "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1-*"
Normalize a model id by removing dates and versions.
Example:
- "gpt-4" -> "gpt-4"
- "gpt-4-20241022" -> "gpt-4"
- "gpt-4-v1.0" -> "gpt-4"
- "gpt-4-20241022-v1.0" -> "gpt-4"
- "gpt-4-20241022-v1.0-beta" -> "gpt-4"
- "gpt-4-20241022-v1.0-beta-1" -> "gpt-4"
- "gpt-4-20241022-v1.0-beta-1" -> "gpt-4"

Args:
model_id: The original model ID
model_id: The model id to normalize

Returns:
The glob version of the model name
The normalized model id
"""
# Pattern to match various date and version formats
# Matches:
# - YYYYMMDD (e.g., 20250522)
# - YYYY-MM-DD (e.g., 2025-06-10)
# - YYYY/MM/DD (e.g., 2025/06/10)
# - YYYY.MM.DD (e.g., 2025.06.10)
# - v followed by version numbers (e.g., v1:0, v2.1, v3)
# - @ followed by dates (e.g., @20241022)
# - -v followed by version numbers (e.g., -v1.0)
# - _v followed by version numbers (e.g., _v1.0)

# Use a single comprehensive regex that handles all patterns
# This regex matches:
# 1. Date patterns: -YYYYMMDD, -YYYY-MM-DD, -YYYY/MM/DD, -YYYY.MM.DD
# 2. Version patterns: -v1.0, -v1:0, _v1.0, _v1:0
# 3. @date patterns: @YYYYMMDD, @YYYY-MM-DD, @YYYY/MM/DD, @YYYY.MM.DD
# 4. Combined patterns: -YYYYMMDD-v1:0, @YYYYMMDD-v1:0

# First, handle @date patterns (they have special handling)
glob_name = re.sub(
r"@(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", "@*", model_id
)

# Then handle regular date and version patterns
glob_name = re.sub(
r"([-_])(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", r"\1*", glob_name
return re.sub(
r"(([-_@])(\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8}))?([-_]v\d+[:.]?\d*([-:].*)?)?$", "", model_id
)

# Handle standalone version patterns (without dates)
glob_name = re.sub(r"([-_])v\d+(?:[.:]\d+)*", r"\1*", glob_name)

return glob_name


def _create_prefix_glob_model_name(model_id: str) -> str:
"""
Expand All @@ -87,8 +60,6 @@ def _create_prefix_glob_model_name(model_id: str) -> str:
- "gpt-4" -> "*gpt-4"
- "claude-3-5-sonnet" -> "*claude-3-5-sonnet"
- "o3-pro" -> "*o3-pro"
- "gpt-4o-mini-*" -> "*gpt-4o-mini-*"
- "model@*" -> "*model@*"

Args:
model_id: The original model ID or a suffix-globbed model name
Expand All @@ -104,13 +75,10 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None:
"""
Add glob versions of model names to the models dictionary.

For each model, creates three types of glob versions:
1. Suffix pattern: stripping dates and versions (e.g., "model-20241022" -> "model-*")
2. Prefix pattern: adding wildcard prefix (e.g., "gpt-4" -> "*gpt-4")
3. Prefix-suffix pattern: both wildcards for suffix-globbed names (e.g., "model-*" -> "*model-*")
For each model, it creates a normalized model name, and a prefix glob version of
the model name.


We DO NOT want to have prefix-suffix glob patterns for models that don't have a suffix glob pattern
because that would result in too fuzzy matching, e.g. "gpt-4" -> "*gpt-4*" would match "gpt-4o-mini".

Args:
models_dict: The dictionary of models to add glob versions to
Expand All @@ -120,20 +88,13 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None:
model_ids = list(models_dict.keys())

for model_id in model_ids:
# Add suffix glob pattern (strip dates/versions)
suffix_glob_name = _create_suffix_glob_model_name(model_id)
if suffix_glob_name != model_id and suffix_glob_name not in models_dict:
models_dict[suffix_glob_name] = models_dict[model_id]

# Add prefix-suffix glob pattern (both wildcards) only for models that have a suffix glob
prefix_suffix_glob_name = _create_prefix_glob_model_name(suffix_glob_name)
if prefix_suffix_glob_name not in models_dict:
models_dict[prefix_suffix_glob_name] = models_dict[model_id]

# Add prefix glob pattern (wildcard prefix)
prefix_glob_name = _create_prefix_glob_model_name(model_id)
normalized_model_id = _normalize_model_id(model_id)
if normalized_model_id != model_id and normalized_model_id not in models_dict:
models_dict[normalized_model_id] = models_dict[model_id]

prefix_glob_name = _create_prefix_glob_model_name(normalized_model_id)
if prefix_glob_name not in models_dict:
models_dict[prefix_glob_name] = models_dict[model_id]
models_dict[prefix_glob_name] = models_dict[normalized_model_id]


@instrumented_task(
Expand Down
124 changes: 60 additions & 64 deletions tests/sentry/tasks/test_ai_agent_monitoring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,9 +482,9 @@ def test_fetch_ai_model_costs_custom_model_mapping(self) -> None:
assert "nonexistent-mapping" not in models

@responses.activate
def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
"""Test that glob versions of model names are added correctly"""
# Mock responses with models that should generate glob patterns
def test_fetch_ai_model_costs_with_normalized_and_prefix_glob_names(self) -> None:
"""Test that normalized and prefix glob versions of model names are added correctly"""
# Mock responses with models that have dates/versions that should be normalized
mock_openrouter_response = {
"data": [
{
Expand All @@ -502,7 +502,7 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
},
},
{
"id": "openai/gpt-4", # No date/version, should not generate glob
"id": "openai/gpt-4", # No date/version, normalized version same as original
"pricing": {
"prompt": "0.0000003",
"completion": "0.00000165",
Expand Down Expand Up @@ -548,104 +548,100 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
assert "claude-3-5-haiku@20241022" in models
assert "o3-pro-2025-06-10" in models

# Check suffix glob versions were added
assert "gpt-4o-mini-*" in models
assert "claude-3-5-sonnet-*" in models
assert "claude-3-5-haiku@*" in models
assert "o3-pro-*" in models
# Check normalized versions were added (dates/versions removed)
assert "gpt-4o-mini" in models
assert "claude-3-5-sonnet" in models
assert "claude-3-5-haiku" in models # @ is not part of the date pattern
assert "o3-pro" in models

# Check prefix glob versions were added
assert "*gpt-4o-mini-20250522" in models
assert "*claude-3-5-sonnet-20241022" in models
# Check prefix glob versions of normalized models were added
assert "*gpt-4o-mini" in models
assert "*claude-3-5-sonnet" in models
assert "*gpt-4" in models
assert "*claude-3-5-haiku@20241022" in models
assert "*o3-pro-2025-06-10" in models
assert "*claude-3-5-haiku" in models
assert "*o3-pro" in models

# Check prefix-suffix glob versions were added (only for models with suffix globs)
assert "*gpt-4o-mini-*" in models
assert "*claude-3-5-sonnet-*" in models
assert "*claude-3-5-haiku@*" in models
assert "*o3-pro-*" in models

# Verify glob versions have same pricing as original models
# Verify normalized versions have same pricing as original models
gpt4o_mini_original = models["gpt-4o-mini-20250522"]
gpt4o_mini_glob = models["gpt-4o-mini-*"]
assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_glob.get("inputPerToken")
assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_glob.get("outputPerToken")
gpt4o_mini_normalized = models["gpt-4o-mini"]
assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_normalized.get(
"inputPerToken"
)
assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_normalized.get(
"outputPerToken"
)

claude_sonnet_original = models["claude-3-5-sonnet-20241022"]
claude_sonnet_glob = models["claude-3-5-sonnet-*"]
assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_glob.get(
claude_sonnet_normalized = models["claude-3-5-sonnet"]
assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_normalized.get(
"inputPerToken"
)
assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_glob.get(
assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_normalized.get(
"outputPerToken"
)

claude_haiku_original = models["claude-3-5-haiku@20241022"]
claude_haiku_glob = models["claude-3-5-haiku@*"]
assert claude_haiku_original.get("inputPerToken") == claude_haiku_glob.get("inputPerToken")
assert claude_haiku_original.get("outputPerToken") == claude_haiku_glob.get(
claude_haiku_normalized = models["claude-3-5-haiku"]
assert claude_haiku_original.get("inputPerToken") == claude_haiku_normalized.get(
"inputPerToken"
)
assert claude_haiku_original.get("outputPerToken") == claude_haiku_normalized.get(
"outputPerToken"
)

o3_pro_original = models["o3-pro-2025-06-10"]
o3_pro_glob = models["o3-pro-*"]
assert o3_pro_original.get("inputPerToken") == o3_pro_glob.get("inputPerToken")
assert o3_pro_original.get("outputPerToken") == o3_pro_glob.get("outputPerToken")
o3_pro_normalized = models["o3-pro"]
assert o3_pro_original.get("inputPerToken") == o3_pro_normalized.get("inputPerToken")
assert o3_pro_original.get("outputPerToken") == o3_pro_normalized.get("outputPerToken")

# Verify gpt-4 (no date/version) doesn't have a suffix glob version
assert "gpt-4*" not in models

# Verify prefix glob versions have same pricing as original models
gpt4_original = models["gpt-4"]
# Verify prefix glob versions have same pricing as normalized models
gpt4_normalized = models["gpt-4"]
gpt4_prefix_glob = models["*gpt-4"]
assert gpt4_original.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken")
assert gpt4_original.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken")
assert gpt4_normalized.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken")
assert gpt4_normalized.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken")

# Verify prefix-suffix glob versions have same pricing as original models
gpt4o_mini_prefix_suffix_glob = models["*gpt-4o-mini-*"]
assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_prefix_suffix_glob.get(
gpt4o_mini_prefix_glob = models["*gpt-4o-mini"]
assert gpt4o_mini_normalized.get("inputPerToken") == gpt4o_mini_prefix_glob.get(
"inputPerToken"
)
assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_prefix_suffix_glob.get(
assert gpt4o_mini_normalized.get("outputPerToken") == gpt4o_mini_prefix_glob.get(
"outputPerToken"
)

@responses.activate
def test_create_suffix_glob_model_name_various_formats(self) -> None:
"""Test suffix glob generation with various date and version formats"""
from sentry.tasks.ai_agent_monitoring import _create_suffix_glob_model_name
def test_normalize_model_id(self) -> None:
"""Test model ID normalization with various date and version formats"""
from sentry.tasks.ai_agent_monitoring import _normalize_model_id

# Test cases with expected outputs
test_cases = [
("model-20250522", "model-*"), # YYYYMMDD -> *
("model-2025-06-10", "model-*"), # YYYY-MM-DD -> *
("model-2025/06/10", "model-*"), # YYYY/MM/DD -> *
("model-2025.06.10", "model-*"), # YYYY.MM.DD -> *
("model-v1.0", "model-*"), # v1.0 -> *
("model-v2.1.0", "model-*"), # v2.1.0 -> *
("model@20241022", "model@*"), # @YYYYMMDD -> @*
("model-v1:0", "model-*"), # v1:0 -> *
("model-20250610-v1:0", "model-*"), # YYYYMMDD-v1:0 -> *
("model@20250610-v1:0", "model@*"), # @YYYYMMDD-v1:0 -> @*
("model-20250522", "model"), # YYYYMMDD removed
("model-2025-06-10", "model"), # YYYY-MM-DD removed
("model-2025/06/10", "model"), # YYYY/MM/DD removed
("model-2025.06.10", "model"), # YYYY.MM.DD removed
("model-v1.0", "model"), # v1.0 removed
("model@20241022", "model"), # @YYYYMMDD removed
("model-v1:0", "model"), # v1:0 removed
("model-20250610-v1:0", "model"), # YYYYMMDD-v1:0 removed
("model@20250610-v1:0", "model"), # @YYYYMMDD-v1:0 removed
("gpt-4", "gpt-4"), # No date/version, unchanged
("claude-3-5-sonnet", "claude-3-5-sonnet"), # Numbers are part of model name, unchanged
]

for model_id, expected_glob in test_cases:
actual_glob = _create_suffix_glob_model_name(model_id)
for model_id, expected_normalized in test_cases:
actual_normalized = _normalize_model_id(model_id)
assert (
actual_glob == expected_glob
), f"Expected {expected_glob} for {model_id}, got {actual_glob}"
actual_normalized == expected_normalized
), f"Expected {expected_normalized} for {model_id}, got {actual_normalized}"

@responses.activate
def test_create_prefix_glob_model_name(self) -> None:
"""Test prefix glob generation for model names"""
from sentry.tasks.ai_agent_monitoring import _create_prefix_glob_model_name

# Test cases with expected outputs
test_cases = [
("gpt-4", "*gpt-4"),
("gpt-4o-mini-*", "*gpt-4o-mini-*"),
("gpt-4o-mini", "*gpt-4o-mini"),
("claude-3-5-sonnet", "*claude-3-5-sonnet"),
("", "*"),
]

Expand Down
Loading