From afaffdd3c440407c198aa7192b97cc77f32146dc Mon Sep 17 00:00:00 2001 From: Vjeran Grozdanic Date: Tue, 18 Nov 2025 10:13:28 +0100 Subject: [PATCH] feat(ai): Normalize model names for a better cost calculation --- src/sentry/tasks/ai_agent_monitoring.py | 85 ++++-------- .../sentry/tasks/test_ai_agent_monitoring.py | 124 +++++++++--------- 2 files changed, 83 insertions(+), 126 deletions(-) diff --git a/src/sentry/tasks/ai_agent_monitoring.py b/src/sentry/tasks/ai_agent_monitoring.py index c0a6a5adc90891..8172efc277db44 100644 --- a/src/sentry/tasks/ai_agent_monitoring.py +++ b/src/sentry/tasks/ai_agent_monitoring.py @@ -26,55 +26,28 @@ MODELS_DEV_API_URL = "https://models.dev/api.json" -def _create_suffix_glob_model_name(model_id: str) -> str: +def _normalize_model_id(model_id: str) -> str: """ - Create a suffix glob version of a model name by stripping dates and versions. - - Examples: - - "claude-4-sonnet-20250522" -> "claude-4-sonnet-*" - - "o3-pro-2025-06-10" -> "o3-pro-*" - - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku@*" - - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1-*" + Normalize a model id by removing dates and versions. + Example: + - "gpt-4" -> "gpt-4" + - "gpt-4-20241022" -> "gpt-4" + - "gpt-4-v1.0" -> "gpt-4" + - "gpt-4-20241022-v1.0" -> "gpt-4" + - "gpt-4-20241022-v1.0-beta" -> "gpt-4" + - "gpt-4-20241022-v1.0-beta-1" -> "gpt-4" + - "gpt-4-20241022-v1.0-beta-1" -> "gpt-4" Args: - model_id: The original model ID + model_id: The model id to normalize Returns: - The glob version of the model name + The normalized model id """ - # Pattern to match various date and version formats - # Matches: - # - YYYYMMDD (e.g., 20250522) - # - YYYY-MM-DD (e.g., 2025-06-10) - # - YYYY/MM/DD (e.g., 2025/06/10) - # - YYYY.MM.DD (e.g., 2025.06.10) - # - v followed by version numbers (e.g., v1:0, v2.1, v3) - # - @ followed by dates (e.g., @20241022) - # - -v followed by version numbers (e.g., -v1.0) - # - _v followed by version numbers (e.g., _v1.0) - - # Use a single comprehensive regex that handles all patterns - # This regex matches: - # 1. Date patterns: -YYYYMMDD, -YYYY-MM-DD, -YYYY/MM/DD, -YYYY.MM.DD - # 2. Version patterns: -v1.0, -v1:0, _v1.0, _v1:0 - # 3. @date patterns: @YYYYMMDD, @YYYY-MM-DD, @YYYY/MM/DD, @YYYY.MM.DD - # 4. Combined patterns: -YYYYMMDD-v1:0, @YYYYMMDD-v1:0 - - # First, handle @date patterns (they have special handling) - glob_name = re.sub( - r"@(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", "@*", model_id - ) - - # Then handle regular date and version patterns - glob_name = re.sub( - r"([-_])(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", r"\1*", glob_name + return re.sub( + r"(([-_@])(\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8}))?([-_]v\d+[:.]?\d*([-:].*)?)?$", "", model_id ) - # Handle standalone version patterns (without dates) - glob_name = re.sub(r"([-_])v\d+(?:[.:]\d+)*", r"\1*", glob_name) - - return glob_name - def _create_prefix_glob_model_name(model_id: str) -> str: """ @@ -87,8 +60,6 @@ def _create_prefix_glob_model_name(model_id: str) -> str: - "gpt-4" -> "*gpt-4" - "claude-3-5-sonnet" -> "*claude-3-5-sonnet" - "o3-pro" -> "*o3-pro" - - "gpt-4o-mini-*" -> "*gpt-4o-mini-*" - - "model@*" -> "*model@*" Args: model_id: The original model ID or a suffix-globbed model name @@ -104,13 +75,10 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None: """ Add glob versions of model names to the models dictionary. - For each model, creates three types of glob versions: - 1. Suffix pattern: stripping dates and versions (e.g., "model-20241022" -> "model-*") - 2. Prefix pattern: adding wildcard prefix (e.g., "gpt-4" -> "*gpt-4") - 3. Prefix-suffix pattern: both wildcards for suffix-globbed names (e.g., "model-*" -> "*model-*") + For each model, it creates a normalized model name, and a prefix glob version of + the model name. + - We DO NOT want to have prefix-suffix glob patterns for models that don't have a suffix glob pattern - because that would result in too fuzzy matching, e.g. "gpt-4" -> "*gpt-4*" would match "gpt-4o-mini". Args: models_dict: The dictionary of models to add glob versions to @@ -120,20 +88,13 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None: model_ids = list(models_dict.keys()) for model_id in model_ids: - # Add suffix glob pattern (strip dates/versions) - suffix_glob_name = _create_suffix_glob_model_name(model_id) - if suffix_glob_name != model_id and suffix_glob_name not in models_dict: - models_dict[suffix_glob_name] = models_dict[model_id] - - # Add prefix-suffix glob pattern (both wildcards) only for models that have a suffix glob - prefix_suffix_glob_name = _create_prefix_glob_model_name(suffix_glob_name) - if prefix_suffix_glob_name not in models_dict: - models_dict[prefix_suffix_glob_name] = models_dict[model_id] - - # Add prefix glob pattern (wildcard prefix) - prefix_glob_name = _create_prefix_glob_model_name(model_id) + normalized_model_id = _normalize_model_id(model_id) + if normalized_model_id != model_id and normalized_model_id not in models_dict: + models_dict[normalized_model_id] = models_dict[model_id] + + prefix_glob_name = _create_prefix_glob_model_name(normalized_model_id) if prefix_glob_name not in models_dict: - models_dict[prefix_glob_name] = models_dict[model_id] + models_dict[prefix_glob_name] = models_dict[normalized_model_id] @instrumented_task( diff --git a/tests/sentry/tasks/test_ai_agent_monitoring.py b/tests/sentry/tasks/test_ai_agent_monitoring.py index 39c10c08bb9a47..80e1a4af52b3bf 100644 --- a/tests/sentry/tasks/test_ai_agent_monitoring.py +++ b/tests/sentry/tasks/test_ai_agent_monitoring.py @@ -482,9 +482,9 @@ def test_fetch_ai_model_costs_custom_model_mapping(self) -> None: assert "nonexistent-mapping" not in models @responses.activate - def test_fetch_ai_model_costs_with_glob_model_names(self) -> None: - """Test that glob versions of model names are added correctly""" - # Mock responses with models that should generate glob patterns + def test_fetch_ai_model_costs_with_normalized_and_prefix_glob_names(self) -> None: + """Test that normalized and prefix glob versions of model names are added correctly""" + # Mock responses with models that have dates/versions that should be normalized mock_openrouter_response = { "data": [ { @@ -502,7 +502,7 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None: }, }, { - "id": "openai/gpt-4", # No date/version, should not generate glob + "id": "openai/gpt-4", # No date/version, normalized version same as original "pricing": { "prompt": "0.0000003", "completion": "0.00000165", @@ -548,96 +548,91 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None: assert "claude-3-5-haiku@20241022" in models assert "o3-pro-2025-06-10" in models - # Check suffix glob versions were added - assert "gpt-4o-mini-*" in models - assert "claude-3-5-sonnet-*" in models - assert "claude-3-5-haiku@*" in models - assert "o3-pro-*" in models + # Check normalized versions were added (dates/versions removed) + assert "gpt-4o-mini" in models + assert "claude-3-5-sonnet" in models + assert "claude-3-5-haiku" in models # @ is not part of the date pattern + assert "o3-pro" in models - # Check prefix glob versions were added - assert "*gpt-4o-mini-20250522" in models - assert "*claude-3-5-sonnet-20241022" in models + # Check prefix glob versions of normalized models were added + assert "*gpt-4o-mini" in models + assert "*claude-3-5-sonnet" in models assert "*gpt-4" in models - assert "*claude-3-5-haiku@20241022" in models - assert "*o3-pro-2025-06-10" in models + assert "*claude-3-5-haiku" in models + assert "*o3-pro" in models - # Check prefix-suffix glob versions were added (only for models with suffix globs) - assert "*gpt-4o-mini-*" in models - assert "*claude-3-5-sonnet-*" in models - assert "*claude-3-5-haiku@*" in models - assert "*o3-pro-*" in models - - # Verify glob versions have same pricing as original models + # Verify normalized versions have same pricing as original models gpt4o_mini_original = models["gpt-4o-mini-20250522"] - gpt4o_mini_glob = models["gpt-4o-mini-*"] - assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_glob.get("inputPerToken") - assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_glob.get("outputPerToken") + gpt4o_mini_normalized = models["gpt-4o-mini"] + assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_normalized.get( + "inputPerToken" + ) + assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_normalized.get( + "outputPerToken" + ) claude_sonnet_original = models["claude-3-5-sonnet-20241022"] - claude_sonnet_glob = models["claude-3-5-sonnet-*"] - assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_glob.get( + claude_sonnet_normalized = models["claude-3-5-sonnet"] + assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_normalized.get( "inputPerToken" ) - assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_glob.get( + assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_normalized.get( "outputPerToken" ) claude_haiku_original = models["claude-3-5-haiku@20241022"] - claude_haiku_glob = models["claude-3-5-haiku@*"] - assert claude_haiku_original.get("inputPerToken") == claude_haiku_glob.get("inputPerToken") - assert claude_haiku_original.get("outputPerToken") == claude_haiku_glob.get( + claude_haiku_normalized = models["claude-3-5-haiku"] + assert claude_haiku_original.get("inputPerToken") == claude_haiku_normalized.get( + "inputPerToken" + ) + assert claude_haiku_original.get("outputPerToken") == claude_haiku_normalized.get( "outputPerToken" ) o3_pro_original = models["o3-pro-2025-06-10"] - o3_pro_glob = models["o3-pro-*"] - assert o3_pro_original.get("inputPerToken") == o3_pro_glob.get("inputPerToken") - assert o3_pro_original.get("outputPerToken") == o3_pro_glob.get("outputPerToken") + o3_pro_normalized = models["o3-pro"] + assert o3_pro_original.get("inputPerToken") == o3_pro_normalized.get("inputPerToken") + assert o3_pro_original.get("outputPerToken") == o3_pro_normalized.get("outputPerToken") - # Verify gpt-4 (no date/version) doesn't have a suffix glob version - assert "gpt-4*" not in models - - # Verify prefix glob versions have same pricing as original models - gpt4_original = models["gpt-4"] + # Verify prefix glob versions have same pricing as normalized models + gpt4_normalized = models["gpt-4"] gpt4_prefix_glob = models["*gpt-4"] - assert gpt4_original.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken") - assert gpt4_original.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken") + assert gpt4_normalized.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken") + assert gpt4_normalized.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken") - # Verify prefix-suffix glob versions have same pricing as original models - gpt4o_mini_prefix_suffix_glob = models["*gpt-4o-mini-*"] - assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_prefix_suffix_glob.get( + gpt4o_mini_prefix_glob = models["*gpt-4o-mini"] + assert gpt4o_mini_normalized.get("inputPerToken") == gpt4o_mini_prefix_glob.get( "inputPerToken" ) - assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_prefix_suffix_glob.get( + assert gpt4o_mini_normalized.get("outputPerToken") == gpt4o_mini_prefix_glob.get( "outputPerToken" ) - @responses.activate - def test_create_suffix_glob_model_name_various_formats(self) -> None: - """Test suffix glob generation with various date and version formats""" - from sentry.tasks.ai_agent_monitoring import _create_suffix_glob_model_name + def test_normalize_model_id(self) -> None: + """Test model ID normalization with various date and version formats""" + from sentry.tasks.ai_agent_monitoring import _normalize_model_id # Test cases with expected outputs test_cases = [ - ("model-20250522", "model-*"), # YYYYMMDD -> * - ("model-2025-06-10", "model-*"), # YYYY-MM-DD -> * - ("model-2025/06/10", "model-*"), # YYYY/MM/DD -> * - ("model-2025.06.10", "model-*"), # YYYY.MM.DD -> * - ("model-v1.0", "model-*"), # v1.0 -> * - ("model-v2.1.0", "model-*"), # v2.1.0 -> * - ("model@20241022", "model@*"), # @YYYYMMDD -> @* - ("model-v1:0", "model-*"), # v1:0 -> * - ("model-20250610-v1:0", "model-*"), # YYYYMMDD-v1:0 -> * - ("model@20250610-v1:0", "model@*"), # @YYYYMMDD-v1:0 -> @* + ("model-20250522", "model"), # YYYYMMDD removed + ("model-2025-06-10", "model"), # YYYY-MM-DD removed + ("model-2025/06/10", "model"), # YYYY/MM/DD removed + ("model-2025.06.10", "model"), # YYYY.MM.DD removed + ("model-v1.0", "model"), # v1.0 removed + ("model@20241022", "model"), # @YYYYMMDD removed + ("model-v1:0", "model"), # v1:0 removed + ("model-20250610-v1:0", "model"), # YYYYMMDD-v1:0 removed + ("model@20250610-v1:0", "model"), # @YYYYMMDD-v1:0 removed + ("gpt-4", "gpt-4"), # No date/version, unchanged + ("claude-3-5-sonnet", "claude-3-5-sonnet"), # Numbers are part of model name, unchanged ] - for model_id, expected_glob in test_cases: - actual_glob = _create_suffix_glob_model_name(model_id) + for model_id, expected_normalized in test_cases: + actual_normalized = _normalize_model_id(model_id) assert ( - actual_glob == expected_glob - ), f"Expected {expected_glob} for {model_id}, got {actual_glob}" + actual_normalized == expected_normalized + ), f"Expected {expected_normalized} for {model_id}, got {actual_normalized}" - @responses.activate def test_create_prefix_glob_model_name(self) -> None: """Test prefix glob generation for model names""" from sentry.tasks.ai_agent_monitoring import _create_prefix_glob_model_name @@ -645,7 +640,8 @@ def test_create_prefix_glob_model_name(self) -> None: # Test cases with expected outputs test_cases = [ ("gpt-4", "*gpt-4"), - ("gpt-4o-mini-*", "*gpt-4o-mini-*"), + ("gpt-4o-mini", "*gpt-4o-mini"), + ("claude-3-5-sonnet", "*claude-3-5-sonnet"), ("", "*"), ]