diff --git a/src/coding/proxy/config/config.default.yaml b/src/coding/proxy/config/config.default.yaml index 05cbf4a..8d34914 100644 --- a/src/coding/proxy/config/config.default.yaml +++ b/src/coding/proxy/config/config.default.yaml @@ -122,9 +122,11 @@ failover: error_types: - "rate_limit_error" - "overloaded_error" + - "api_error" error_message_patterns: - "quota" - "usage cap" + - "internal network failure" # === 模型映射 === model_mapping: diff --git a/src/coding/proxy/config/resiliency.py b/src/coding/proxy/config/resiliency.py index 71ba258..751625b 100644 --- a/src/coding/proxy/config/resiliency.py +++ b/src/coding/proxy/config/resiliency.py @@ -30,7 +30,7 @@ class FailoverConfig(BaseModel): default=["rate_limit_error", "overloaded_error", "api_error"], ) error_message_patterns: list[str] = Field( - default=["quota", "limit exceeded", "usage cap", "capacity"], + default=["quota", "limit exceeded", "usage cap", "capacity", "internal network failure"], ) diff --git a/src/coding/proxy/routing/executor.py b/src/coding/proxy/routing/executor.py index 8aa67ab..7d25788 100644 --- a/src/coding/proxy/routing/executor.py +++ b/src/coding/proxy/routing/executor.py @@ -451,13 +451,10 @@ async def execute_message( failed_tier_name = tier.name continue - if not is_last and tier.vendor.should_trigger_failover( + if tier.vendor.should_trigger_failover( resp.status_code, {"error": {"type": resp.error_type, "message": resp.error_message}}, ): - logger.warning( - "Tier %s error %d, failing over", tier.name, resp.status_code - ) rl_info = parse_rate_limit_headers( resp.response_headers, resp.status_code, resp.error_message ) @@ -466,8 +463,14 @@ async def execute_message( retry_after_seconds=compute_effective_retry_seconds(rl_info), rate_limit_deadline=compute_rate_limit_deadline(rl_info), ) - failed_tier_name = tier.name - continue + if not is_last: + logger.warning( + "Tier %s error %d, failing over", + tier.name, + resp.status_code, + ) + failed_tier_name = tier.name + continue # 最后一层或不可 failover 的错误：记录并返回原始响应 _log_vendor_response_error(tier.name, resp, body, is_stream=False) diff --git a/src/coding/proxy/vendors/base.py b/src/coding/proxy/vendors/base.py index 975d3af..0c08248 100644 --- a/src/coding/proxy/vendors/base.py +++ b/src/coding/proxy/vendors/base.py @@ -268,8 +268,8 @@ def should_trigger_failover( for pattern in self._failover_config.error_message_patterns: if pattern.lower() in error_message: return True - # 429/503/529 即使无法解析 body 也触发故障转移 - return status_code in (429, 503, 529) + # 429/500/503/529 即使无法解析 body 也触发故障转移 + return status_code in (429, 500, 503, 529) async def check_health(self) -> bool: """检查供应商健康状态（轻量级探测）. diff --git a/tests/test_request_normalizer.py b/tests/test_request_normalizer.py index 023f679..7a7f7ed 100644 --- a/tests/test_request_normalizer.py +++ b/tests/test_request_normalizer.py @@ -89,480 +89,3 @@ def test_unknown_tool_result_id_marks_fatal_reason(): assert result.recoverable is False assert result.fatal_reasons - - -<<<<<<< HEAD -class TestRelocateMisplacedToolResults: - """:func:`_relocate_misplaced_tool_results` 测试 — 覆盖 Anthropic 400 - ``tool_result can only be in user messages`` 错误的修复场景. - """ - - def test_relocates_tool_result_from_assistant_to_user(self): - """assistant 消息中的 tool_result 应被迁移到最近的前置 user 消息.""" -======= -# ── 跨供应商 tool_result 位置错位修复测试 ────────────────────── - - -class TestMisplacedToolResultRepair: - """验证 tool_result 出现在非 user 消息中时被修复到正确位置. - - 典型触发场景：Zhipu GLM-5 通过 Anthropic 兼容端点返回的 assistant 响应中 - 同时包含 tool_use 和 tool_result，Claude Code 将其存入对话历史后， - 后续请求的 assistant message 中包含 tool_result。 - Anthropic API 严格要求 tool_result 只能出现在 user 消息中。 - """ - - def test_repairs_tool_result_from_assistant_to_user(self): - """assistant 消息中的 tool_result 应被移到紧随其后的 user 消息.""" ->>>>>>> acbcc4b (fix(failover): 修复 Zhipu GLM-5 跨供应商回退时 tool_result 角色错位导致级联故障;) - result = normalize_anthropic_request( - { - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "hello"}]}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "let me check"}, - { - "type": "tool_result", - "tool_use_id": "toolu_01", - "content": "result data", - }, - ], - }, - { - "role": "user", - "content": "thanks", - }, - ], - } - ) - - assert result.recoverable is True -<<<<<<< HEAD - msgs = result.body["messages"] - # assistant 消息中不应再有 tool_result - assistant_content = msgs[1]["content"] - assert not any( - b.get("type") == "tool_result" for b in assistant_content if isinstance(b, dict) - ) - # tool_result 应出现在 user 消息中 - user_content = msgs[0]["content"] - tool_results = [ - b for b in user_content if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - assert tool_results[0]["tool_use_id"] == "toolu_01" - assert any("tool_result_relocated" in a for a in result.adaptations) - - def test_relocates_multiple_tool_results_from_assistant(self): - """assistant 消息中的多个 tool_result 块应全部迁移.""" -======= - # assistant 消息应只保留 tool_use - assistant_content = result.body["messages"][1]["content"] - assert len(assistant_content) == 1 - assert assistant_content[0]["type"] == "tool_use" - assert assistant_content[0]["id"] == "toolu_123" - # tool_result 应被移到紧随其后的 user 消息 - user_content = result.body["messages"][2]["content"] - # user 消息原有内容 "thanks" + 修复追加的 tool_result - assert any( - isinstance(b, dict) and b.get("type") == "tool_result" - for b in user_content - if isinstance(b, dict) - ) - tool_result_block = next( - b for b in user_content if isinstance(b, dict) and b.get("type") == "tool_result" - ) - assert tool_result_block["tool_use_id"] == "toolu_123" - assert tool_result_block["content"] == "file1.txt\nfile2.txt" - assert "misplaced_tool_result_repaired" in result.adaptations - - def test_repairs_preserves_other_blocks(self): - """修复 tool_result 时保留同消息中的其他内容块.""" ->>>>>>> acbcc4b (fix(failover): 修复 Zhipu GLM-5 跨供应商回退时 tool_result 角色错位导致级联故障;) - result = normalize_anthropic_request( - { - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "hi"}]}, - { - "role": "assistant", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01", - "content": "result1", - }, - { - "type": "tool_result", - "tool_use_id": "toolu_02", - "content": "result2", - }, - ], - }, - { - "role": "user", - "content": "next step", - }, - ], - } - ) - - assert result.recoverable is True -<<<<<<< HEAD - user_content = result.body["messages"][0]["content"] - tool_results = [ - b for b in user_content if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 2 -======= - assistant_content = result.body["messages"][0]["content"] - assert len(assistant_content) == 3 - types = [b["type"] for b in assistant_content] - assert types == ["text", "tool_use", "text"] - # tool_result 被移到 user 消息 - user_content = result.body["messages"][1]["content"] - tool_result_blocks = [ - b for b in user_content if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_result_blocks) == 1 - assert "misplaced_tool_result_repaired" in result.adaptations ->>>>>>> acbcc4b (fix(failover): 修复 Zhipu GLM-5 跨供应商回退时 tool_result 角色错位导致级联故障;) - - def test_creates_new_user_message_when_no_preceding_user(self): - """无前置 user 消息时，应在头部创建新 user 消息容纳错位 tool_result.""" - result = normalize_anthropic_request( - { - "messages": [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "thinking..."}, - { - "type": "tool_result", - "tool_use_id": "toolu_01", - "content": "orphan result", - }, - ], - }, - ], - } - ) - - assert result.recoverable is True - msgs = result.body["messages"] - # 新创建的 user 消息应在索引 0 - assert msgs[0]["role"] == "user" - new_user_content = msgs[0]["content"] - tool_results = [ - b - for b in new_user_content - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - assert tool_results[0]["tool_use_id"] == "toolu_01" - - def test_finds_nearest_user_message_across_multiple_messages(self): - """应跳过中间的 assistant 消息，找到最近的前置 user 消息.""" - result = normalize_anthropic_request( - { - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "first"}]}, - { - "role": "assistant", - "content": [{"type": "tool_use", "id": "tu_1", "name": "bash", "input": {}}], - }, - {"role": "user", "content": [{"type": "text", "text": "second"}]}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "processing"}, - { - "type": "tool_result", - "tool_use_id": "tu_1", - "content": "bash output", - }, - ], - }, - ], - } - ) - - assert result.recoverable is True - # tool_result 应被迁移到 messages[2]（第二个 user 消息），而非 messages[0] - target_user = result.body["messages"][2] - tool_results = [ - b - for b in target_user["content"] - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - # 第一个 user 消息不应有新增的 tool_result - first_user = result.body["messages"][0] - first_trs = [ - b - for b in first_user["content"] - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(first_trs) == 0 - - def test_noop_when_tool_results_already_in_user_messages(self): - """tool_result 已在正确位置时，不应触发迁移逻辑.""" - result = normalize_anthropic_request( - { - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "hello"}]}, - { - "role": "assistant", - "content": [ - {"type": "tool_use", "id": "tu_1", "name": "bash", "input": {}} - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "tu_1", - "content": "ok", - } - ], - }, - ], - } - ) - - assert result.recoverable is True -<<<<<<< HEAD - assert not any("tool_result_relocated" in a for a in result.adaptations) - - def test_preserves_existing_user_message_structure(self): - """迁移不应破坏目标 user 消息的现有内容结构.""" -======= - user_content = result.body["messages"][1]["content"] - assert len(user_content) == 1 - assert user_content[0]["type"] == "tool_result" - assert "misplaced_tool_result_repaired" not in result.adaptations - - def test_duplicate_tool_result_not_duplicated_in_repair(self): - """assistant 和 user 消息中同时有同 tool_use_id 的 tool_result 时， - 修复不会在 user 消息中产生重复.""" ->>>>>>> acbcc4b (fix(failover): 修复 Zhipu GLM-5 跨供应商回退时 tool_result 角色错位导致级联故障;) - result = normalize_anthropic_request( - { - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "original text"}, - {"type": "image", "source": {"type": "base64", "data": "abc", "media_type": "img/png"}}, - ], - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01", - "content": "moved result", - }, - ], - }, - ], - } - ) - - assert result.recoverable is True -<<<<<<< HEAD - user_content = result.body["messages"][0]["content"] - # 原有内容应保留 - assert any(isinstance(b, dict) and b.get("type") == "text" for b in user_content) - assert any(isinstance(b, dict) and b.get("type") == "image" for b in user_content) - # 迁移的 tool_result 应追加在末尾 - tool_results = [ - b for b in user_content if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - - def test_handles_system_role_with_tool_result(self): - """system 角色消息中的 tool_result 也应被迁移.""" - result = normalize_anthropic_request( - { - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "help"}]}, - { - "role": "system", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01", - "content": "sys result", - }, - ], - }, - ], - } - ) - - assert result.recoverable is True - # system 消息中不再有 tool_result - sys_content = result.body["messages"][1]["content"] - assert not any( - b.get("type") == "tool_result" for b in sys_content if isinstance(b, dict) - ) - # tool_result 在 user 消息中 - user_content = result.body["messages"][0]["content"] - assert any( - isinstance(b, dict) and b.get("type") == "tool_result" for b in user_content - ) -======= - # assistant 中的 tool_result 被移除 - assistant_content = result.body["messages"][0]["content"] - assert len(assistant_content) == 1 - assert assistant_content[0]["type"] == "tool_use" - # user 中的 tool_result 保留，但不产生重复 - user_content = result.body["messages"][1]["content"] - tool_result_blocks = [ - b - for b in user_content - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_result_blocks) == 1 - assert tool_result_blocks[0]["content"] == "correct result" - assert "misplaced_tool_result_repaired" in result.adaptations - - def test_creates_user_message_when_none_exists(self): - """当 assistant 后没有 user 消息时，应创建新的 user 消息.""" - result = normalize_anthropic_request( - { - "messages": [ - { - "role": "user", - "content": "run it", - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_200", - "content": "orphaned result", - }, - ], - }, - ], - } - ) - - assert result.recoverable is True - # assistant 消息变为空，应有占位符 - assistant_content = result.body["messages"][1]["content"] - assert len(assistant_content) == 1 - assert assistant_content[0]["type"] == "text" - # 新的 user 消息应被插入 - assert len(result.body["messages"]) == 3 - assert result.body["messages"][2]["role"] == "user" - new_user_content = result.body["messages"][2]["content"] - assert len(new_user_content) == 1 - assert new_user_content[0]["type"] == "tool_result" - assert new_user_content[0]["tool_use_id"] == "toolu_200" - assert "misplaced_tool_result_user_message_created" in result.adaptations - assert "empty_assistant_message_placeholder_added" in result.adaptations - - def test_deep_conversation_with_misplaced_tool_result_at_index_105(self): - """模拟长对话（105+ 消息）中 tool_result 出现在 assistant 消息的场景. - - 重现原始 bug 报告中的错误：messages.105 处 tool_result 位置不合规。 - """ - # 构建一个 106 条消息的对话历史 - messages = [] - for i in range(104): - role = "user" if i % 2 == 0 else "assistant" - messages.append({ - "role": role, - "content": f"message {i}", - }) - - # 在消息 104（assistant）中放置 tool_use + tool_result - messages.append({ - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "toolu_deep_1", - "name": "Bash", - "input": {"command": "find / -name '*.log'"}, - }, - { - "type": "tool_result", - "tool_use_id": "toolu_deep_1", - "content": "/var/log/system.log", - }, - ], - }) - # 消息 105（user） - messages.append({ - "role": "user", - "content": "thanks", - }) - - result = normalize_anthropic_request({"messages": messages}) - - assert result.recoverable is True - # 消息 104 中的 tool_result 应被移除 - assert len(result.body["messages"][104]["content"]) == 1 - assert result.body["messages"][104]["content"][0]["type"] == "tool_use" - # tool_result 应被追加到消息 105 的 user 消息中 - user_msg_105 = result.body["messages"][105] - assert user_msg_105["role"] == "user" - tool_results = [ - b - for b in user_msg_105["content"] - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - assert tool_results[0]["tool_use_id"] == "toolu_deep_1" - assert "misplaced_tool_result_repaired" in result.adaptations - - def test_empty_assistant_message_gets_placeholder(self): - """修复后空 assistant 消息应添加占位 text block.""" - result = normalize_anthropic_request( - { - "messages": [ - { - "role": "user", - "content": "run it", - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_300", - "content": "only block", - }, - ], - }, - { - "role": "user", - "content": "ok", - }, - ], - } - ) - - assert result.recoverable is True - # assistant 消息应有占位符 - assistant_content = result.body["messages"][1]["content"] - assert len(assistant_content) == 1 - assert assistant_content[0]["type"] == "text" - # tool_result 移到 user 消息 - user_content = result.body["messages"][2]["content"] - tool_results = [ - b - for b in user_content - if isinstance(b, dict) and b.get("type") == "tool_result" - ] - assert len(tool_results) == 1 - assert "empty_assistant_message_placeholder_added" in result.adaptations ->>>>>>> acbcc4b (fix(failover): 修复 Zhipu GLM-5 跨供应商回退时 tool_result 角色错位导致级联故障;) diff --git a/tests/test_router_executor.py b/tests/test_router_executor.py index 69b6def..f894704 100644 --- a/tests/test_router_executor.py +++ b/tests/test_router_executor.py @@ -1356,3 +1356,149 @@ async def test_400_without_tool_results_does_not_format_failover(self): # 无 tool_result 的 400 直接返回给客户端（不是最后一层但无下一层可降级） assert resp.status_code == 400 + + +# ── execute_message 最后一层 500 降级记录测试 ───────────────── + + +class TestExecuteMessageLastTier500RecordsFailure: + """非流式路径下终端层（最后一层）vendor 返回 500 时应记录降级状态. + + 覆盖核心修复场景：zhipu 作为终端层返回 500 "Internal Network Failure"， + 即使无法故障转移到下一层，仍需调用 record_failure() 维护降级状态 + （与流式路径 _handle_http_error 的行为对称）。 + """ + + @pytest.mark.asyncio + async def test_last_tier_500_with_failover_records_failure(self): + """最后一层 vendor 返回 500 且 should_trigger_failover=True 时，record_failure 应被调用.""" + from coding.proxy.routing.circuit_breaker import CircuitBreaker + + vendor = _mock_vendor("zhipu") + vendor.send_message = AsyncMock( + return_value=VendorResponse( + status_code=500, + raw_body=b'{"error":{"type":"api_error","message":"Internal Network Failure"}}', + error_type="api_error", + error_message="Internal Network Failure", + ) + ) + vendor.should_trigger_failover.return_value = True + + cb = CircuitBreaker(failure_threshold=3) + tier = _make_tier(vendor, circuit_breaker=cb) + exec_inst = _executor([tier]) + + resp = await exec_inst.execute_message({"model": "claude-haiku-4-5-20251001"}, {}) + + assert resp.status_code == 500 + assert resp.error_message == "Internal Network Failure" + # 验证 record_failure 被调用：CircuitBreaker 的 failure_count 应增加 + assert cb.get_info()["failure_count"] == 1 + + @pytest.mark.asyncio + async def test_last_tier_500_without_failover_no_failure_recorded(self): + """最后一层 vendor 返回 500 但 should_trigger_failover=False 时，不记录降级.""" + from coding.proxy.routing.circuit_breaker import CircuitBreaker + + vendor = _mock_vendor("zhipu") + vendor.send_message = AsyncMock( + return_value=VendorResponse( + status_code=500, + raw_body=b'{"error":{"type":"server_error","message":"unknown"}}', + error_type="server_error", + error_message="unknown", + ) + ) + vendor.should_trigger_failover.return_value = False + + cb = CircuitBreaker(failure_threshold=3) + tier = _make_tier(vendor, circuit_breaker=cb) + exec_inst = _executor([tier]) + + resp = await exec_inst.execute_message({"model": "test"}, {}) + + assert resp.status_code == 500 + # should_trigger_failover=False 时，failure 不应被记录 + assert cb.get_info()["failure_count"] == 0 + + @pytest.mark.asyncio + async def test_last_tier_500_still_returns_response_to_client(self): + """修复后，最后一层 500 仍应正确返回原始错误响应给客户端.""" + vendor = _mock_vendor("zhipu") + error_body = b'{"error":{"type":"api_error","message":"Internal Network Failure"}}' + vendor.send_message = AsyncMock( + return_value=VendorResponse( + status_code=500, + raw_body=error_body, + error_type="api_error", + error_message="Internal Network Failure", + ) + ) + vendor.should_trigger_failover.return_value = True + + exec_inst = _executor([_make_tier(vendor)]) + resp = await exec_inst.execute_message({"model": "test"}, {}) + + assert resp.status_code == 500 + assert resp.raw_body == error_body + assert resp.error_message == "Internal Network Failure" + + @pytest.mark.asyncio + async def test_last_tier_500_with_retry_after_updates_rate_limit(self): + """最后一层 500 含 Retry-After 头时，应更新 tier 的 rate limit deadline.""" + import time + + from coding.proxy.routing.circuit_breaker import CircuitBreaker + + vendor = _mock_vendor("zhipu") + vendor.send_message = AsyncMock( + return_value=VendorResponse( + status_code=429, + raw_body=b'{"error":{"type":"rate_limit_error","message":"rate limited"}}', + error_type="rate_limit_error", + error_message="rate limited", + response_headers={"retry-after": "60"}, + ) + ) + vendor.should_trigger_failover.return_value = True + + tier = _make_tier(vendor, circuit_breaker=CircuitBreaker()) + exec_inst = _executor([tier]) + + resp = await exec_inst.execute_message({"model": "test"}, {}) + + assert resp.status_code == 429 + # 验证 rate limit deadline 被更新 + rl_info = tier.get_rate_limit_info() + assert rl_info["is_rate_limited"] is True + assert rl_info["remaining_seconds"] > 0 + + @pytest.mark.asyncio + async def test_non_last_tier_500_still_failovers(self): + """修复后，非最后一层 500 仍应正常触发故障转移到下一层.""" + bad = _mock_vendor("zhipu") + bad.send_message = AsyncMock( + return_value=VendorResponse( + status_code=500, + raw_body=b'{"error":{"type":"api_error","message":"Internal Network Failure"}}', + error_type="api_error", + error_message="Internal Network Failure", + ) + ) + bad.should_trigger_failover.return_value = True + + good = _mock_vendor("copilot") + good.send_message = AsyncMock( + return_value=VendorResponse( + status_code=200, + raw_body=b'{"content":"ok"}', + usage=UsageInfo(input_tokens=1, output_tokens=1), + ) + ) + + exec_inst = _executor([_make_tier(bad), _make_tier(good)]) + resp = await exec_inst.execute_message({"model": "test"}, {}) + + assert resp.status_code == 200 + assert good.send_message.called diff --git a/tests/test_vendors.py b/tests/test_vendors.py index f02ae91..e9fc241 100644 --- a/tests/test_vendors.py +++ b/tests/test_vendors.py @@ -675,6 +675,43 @@ def test_529_in_failover_config_default(): assert 529 in config.status_codes +# --- 500 Internal Network Failure 降级测试 --- + + +def test_500_api_error_triggers_failover(): + """500 + api_error 应触发降级（FailoverConfig 默认包含 api_error）.""" + anthropic_vendor = AnthropicVendor(AnthropicConfig(), FailoverConfig()) + + assert anthropic_vendor.should_trigger_failover( + 500, {"error": {"type": "api_error", "message": "Internal Network Failure"}} + ) + + +def test_500_internal_network_failure_pattern_triggers_failover(): + """500 + 'internal network failure' 消息应触发降级（默认 patterns 包含该模式）.""" + anthropic_vendor = AnthropicVendor(AnthropicConfig(), FailoverConfig()) + + assert anthropic_vendor.should_trigger_failover( + 500, {"error": {"type": "unknown", "message": "Internal Network Failure"}} + ) + + +def test_500_without_body_triggers_failover(): + """500 无 body 也应触发降级（备用安全网逻辑）.""" + anthropic_vendor = AnthropicVendor(AnthropicConfig(), FailoverConfig()) + + assert anthropic_vendor.should_trigger_failover(500, None) + + +def test_500_in_failover_config_default(): + """验证 FailoverConfig 默认 status_codes 和 error_message_patterns 覆盖 zhipu 500 场景.""" + config = FailoverConfig() + assert 500 in config.status_codes + assert "internal network failure" in [ + p.lower() for p in config.error_message_patterns + ] + + # --- _sanitize_headers_for_synthetic_response ---