Python: Supports better exceptions when Azure OpenAI content filterin…

…g is triggered (#4428)

### Motivation and Context

When an Azure OpenAI content filter error is triggered the current
`AIException` hides the details of that specific error. This makes it
hard for the consumer to see a more detailed description of the error in
their prompt or input.
Solves #4197

### Description

Detects when a content filter error is raised, then parses the
meaningful information about it and raises a new
`ContentFilterAIException` with that information ready for the consumer
to use.

### Contribution Checklist

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

---------

Co-authored-by: Evan Mattson <35585003+moonbox3@users.noreply.github.com>

python/semantic_kernel/connectors/ai/ai_exception.py

@@ -28,6 +28,8 @@ class ErrorCodes(Enum):
         InvalidConfiguration = 8
         # The function is not supported.
         FunctionTypeNotSupported = 9
+        # The LLM raised an error due to improper content.
+        BadContentError = 10
 
     # The error code.
     _error_code: ErrorCodes

python/semantic_kernel/connectors/ai/open_ai/exceptions/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Microsoft. All rights reserved.

python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py

@@ -0,0 +1,112 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict
+
+from openai import BadRequestError
+
+from semantic_kernel.connectors.ai.ai_exception import AIException
+
+
+class ContentFilterResultSeverity(Enum):
+    HIGH = "high"
+    MEDIUM = "medium"
+    SAFE = "safe"
+
+
+@dataclass
+class ContentFilterResult:
+    filtered: bool = False
+    detected: bool = False
+    severity: ContentFilterResultSeverity = ContentFilterResultSeverity.SAFE
+
+    @classmethod
+    def from_inner_error_result(cls, inner_error_results: Dict[str, Any]) -> "ContentFilterResult":
+        """Creates a ContentFilterResult from the inner error results.
+
+        Arguments:
+            key {str} -- The key to get the inner error result from.
+            inner_error_results {Dict[str, Any]} -- The inner error results.
+
+        Returns:
+            ContentFilterResult -- The ContentFilterResult.
+        """
+        return cls(
+            filtered=inner_error_results.get("filtered", False),
+            detected=inner_error_results.get("detected", False),
+            severity=ContentFilterResultSeverity(
+                inner_error_results.get("severity", ContentFilterResultSeverity.SAFE.value)
+            ),
+        )
+
+
+class ContentFilterCodes(Enum):
+    RESPONSIBLE_AI_POLICY_VIOLATION = "ResponsibleAIPolicyViolation"
+
+
+class ContentFilterAIException(AIException):
+    """AI exception for an error from Azure OpenAI's content filter"""
+
+    # The parameter that caused the error.
+    _param: str
+
+    # The error code specific to the content filter.
+    _content_filter_code: ContentFilterCodes
+
+    # The results of the different content filter checks.
+    _content_filter_result: Dict[str, ContentFilterResult]
+
+    def __init__(
+        self,
+        error_code: AIException.ErrorCodes,
+        message: str,
+        inner_exception: BadRequestError,
+    ) -> None:
+        """Initializes a new instance of the ContentFilterAIException class.
+
+        Arguments:
+            error_code {ErrorCodes} -- The error code.
+            message {str} -- The error message.
+            inner_exception {Exception} -- The inner exception.
+        """
+        super().__init__(error_code, message, inner_exception)
+
+        self._param = inner_exception.param
+
+        inner_error = inner_exception.body.get("innererror", {})
+        self._content_filter_code = ContentFilterCodes(inner_error.get("code", ""))
+        self._content_filter_result = dict(
+            [
+                key,
+                ContentFilterResult.from_inner_error_result(values),
+            ]
+            for key, values in inner_error.get("content_filter_result", {}).items()
+        )
+
+    @property
+    def param(self) -> str:
+        """Gets the parameter that caused the error.
+
+        Returns:
+            str -- The parameter that caused the error.
+        """
+        return self._param
+
+    @property
+    def content_filter_code(self) -> ContentFilterCodes:
+        """Gets the error code specific to the content filter.
+
+        Returns:
+            ContentFilterCode -- The error code specific to the content filter.
+        """
+        return self._content_filter_code
+
+    @property
+    def content_filter_result(self) -> Dict[str, ContentFilterResult]:
+        """Gets the result of the content filter checks.
+
+        Returns:
+            Dict[str, ContentFilterResult] -- The result of the content filter checks.
+        """
+        return self._content_filter_result

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py

@@ -5,14 +5,17 @@
 from typing import List, Union
 
 from numpy import array
-from openai import AsyncOpenAI, AsyncStream
+from openai import AsyncOpenAI, AsyncStream, BadRequestError
 from openai.types import Completion
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from pydantic import Field
 
 from semantic_kernel.connectors.ai.ai_exception import AIException
 from semantic_kernel.connectors.ai.ai_request_settings import AIRequestSettings
 from semantic_kernel.connectors.ai.ai_service_client_base import AIServiceClientBase
+from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import (
+    ContentFilterAIException,
+)
 from semantic_kernel.connectors.ai.open_ai.request_settings.open_ai_request_settings import (
     OpenAIEmbeddingRequestSettings,
     OpenAIRequestSettings,
@@ -58,6 +61,18 @@ async def _send_request(
             )
             self.store_usage(response)
             return response
+        except BadRequestError as ex:
+            if ex.code == "content_filter":
+                raise ContentFilterAIException(
+                    AIException.ErrorCodes.BadContentError,
+                    f"{type(self)} service encountered a content error",
+                    ex,
+                )
+            raise AIException(
+                AIException.ErrorCodes.ServiceError,
+                f"{type(self)} service failed to complete the prompt",
+                ex,
+            ) from ex
         except Exception as ex:
             raise AIException(
                 AIException.ErrorCodes.ServiceError,

python/tests/unit/ai/open_ai/services/test_azure_chat_completion.py

@@ -2,7 +2,9 @@
 
 from unittest.mock import AsyncMock, patch
 
+import openai
 import pytest
+from httpx import Request, Response
 from openai import AsyncAzureOpenAI
 from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions
 from pydantic import ValidationError
@@ -17,6 +19,11 @@
 from semantic_kernel.connectors.ai.open_ai.const import (
     USER_AGENT,
 )
+from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import (
+    ContentFilterAIException,
+    ContentFilterCodes,
+    ContentFilterResultSeverity,
+)
 from semantic_kernel.connectors.ai.open_ai.request_settings.azure_chat_request_settings import (
     AzureAISearchDataSources,
     AzureChatRequestSettings,
@@ -440,3 +447,67 @@ async def test_azure_chat_completion_call_with_data_with_parameters_and_Stop_Def
         logit_bias={},
         extra_body=expected_data_settings,
     )
+
+
+CONTENT_FILTERED_ERROR_MESSAGE = (
+    "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please "
+    "modify your prompt and retry. To learn more about our content filtering policies please read our "
+    "documentation: https://go.microsoft.com/fwlink/?linkid=2198766"
+)
+CONTENT_FILTERED_ERROR_FULL_MESSAGE = (
+    "Error code: 400 - {'error': {'message': \"%s\", 'type': null, 'param': 'prompt', 'code': 'content_filter', "
+    "'status': 400, 'innererror': {'code': 'ResponsibleAIPolicyViolation', 'content_filter_result': {'hate': "
+    "{'filtered': True, 'severity': 'high'}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': "
+    "{'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}}}"
+) % CONTENT_FILTERED_ERROR_MESSAGE
+
+
+@pytest.mark.asyncio
+@patch.object(AsyncChatCompletions, "create")
+async def test_azure_chat_completion_content_filtering_raises_correct_exception(
+    mock_create,
+) -> None:
+    deployment_name = "test_deployment"
+    endpoint = "https://test-endpoint.com"
+    api_key = "test_api_key"
+    api_version = "2023-03-15-preview"
+    prompt = "some prompt that would trigger the content filtering"
+    messages = [{"role": "user", "content": prompt}]
+    complete_request_settings = AzureChatRequestSettings()
+
+    mock_create.side_effect = openai.BadRequestError(
+        CONTENT_FILTERED_ERROR_FULL_MESSAGE,
+        response=Response(400, request=Request("POST", endpoint)),
+        body={
+            "message": CONTENT_FILTERED_ERROR_MESSAGE,
+            "type": None,
+            "param": "prompt",
+            "code": "content_filter",
+            "status": 400,
+            "innererror": {
+                "code": "ResponsibleAIPolicyViolation",
+                "content_filter_result": {
+                    "hate": {"filtered": True, "severity": "high"},
+                    "self_harm": {"filtered": False, "severity": "safe"},
+                    "sexual": {"filtered": False, "severity": "safe"},
+                    "violence": {"filtered": False, "severity": "safe"},
+                },
+            },
+        },
+    )
+
+    azure_chat_completion = AzureChatCompletion(
+        deployment_name=deployment_name,
+        endpoint=endpoint,
+        api_key=api_key,
+        api_version=api_version,
+    )
+
+    with pytest.raises(ContentFilterAIException, match="service encountered a content error") as exc_info:
+        await azure_chat_completion.complete_chat_async(messages, complete_request_settings)
+
+    content_filter_exc = exc_info.value
+    assert content_filter_exc.param == "prompt"
+    assert content_filter_exc.content_filter_code == ContentFilterCodes.RESPONSIBLE_AI_POLICY_VIOLATION
+    assert content_filter_exc.content_filter_result["hate"].filtered
+    assert content_filter_exc.content_filter_result["hate"].severity == ContentFilterResultSeverity.HIGH