diff --git a/packages/phoenix-evals/src/phoenix/evals/models/openai.py b/packages/phoenix-evals/src/phoenix/evals/models/openai.py
index 8ef4148bb4..81e792f02d 100644
--- a/packages/phoenix-evals/src/phoenix/evals/models/openai.py
+++ b/packages/phoenix-evals/src/phoenix/evals/models/openai.py
@@ -156,6 +156,11 @@ def _init_open_ai(self) -> None:
             api_key = os.getenv(OPENAI_API_KEY_ENVVAR_NAME)
             if api_key is None:
                 # TODO: Create custom AuthenticationError
+                if self._is_azure:
+                    raise RuntimeError(
+                        "Azure API key not provided. Pass it as an argument to 'api_key' "
+                        "or set it in your environment: 'export OPENAI_API_KEY=****'"
+                    )
                 raise RuntimeError(
                     "OpenAI's API key not provided. Pass it as an argument to 'api_key' "
                     "or set it in your environment: 'export OPENAI_API_KEY=sk-****'"