Python: Update the AzureAIAgent packages, types, and APIs based on Azure SDK's new code (#12084)

### Motivation and Context

The Azure AI Agent Service is moving to GA soon. As part of that,
they've made several changes in the underlying APIs, which require the
use of new packages, updated type imports and some new underlying APIs.
This PR updates the AzureAIAgent to use the latest Azure SDK's new code.

- The client's connection string has been replaced with an endpoint.
- The `AzureAIAgentSettings`'s `model_deployment_name` was updated to
`deployment_name`.
- Tools are now imported from `from azure.ai.agents.models` instead of
`from azure.ai.projects.models`.
- Declarative spec handling for AzureAIAgent was updated based on the
new APIs.
- Unit tests were updated.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

Update the AzureAIAgent related code, samples and tests based on the
Azure SDK's latest APIs.
- Closes #12177 

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

---------

Co-authored-by: Tao Chen <taochen@microsoft.com>

Author: moonbox3

.github/workflows/python-integration-tests.yml

@@ -32,7 +32,7 @@ env:
   AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
   AZURE_OPENAI_AUDIO_TO_TEXT_ENDPOINT: ${{ secrets.AZURE_OPENAI_AUDIO_TO_TEXT_ENDPOINT }}
   AZURE_OPENAI_TEXT_TO_AUDIO_ENDPOINT: ${{ secrets.AZURE_OPENAI_TEXT_TO_AUDIO_ENDPOINT }}
-  AZURE_AI_AGENT_PROJECT_CONNECTION_STRING: ${{ secrets.AZURE_AI_AGENT_PROJECT_CONNECTION_STRING }}
+  AZURE_AI_AGENT_ENDPOINT: ${{ secrets.AZURE_AI_AGENT_ENDPOINT }}
   AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME }}
   AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
   BING_API_KEY: ${{ secrets.BING_API_KEY }}

python/pyproject.toml

@@ -67,9 +67,11 @@ autogen = [
 aws = [
     "boto3>=1.36.4,<1.39.0",
 ]
+# This is temporary to get the PR reviewed, and will be replaced when their new version is public.
 azure = [
+    "azure-ai-projects >= 1.0.0b11",
+    "azure-ai-agents >= 1.0.0",
     "azure-ai-inference >= 1.0.0b6",
-    "azure-ai-projects >= 1.0.0b7, <1.0.0b11",
     "azure-core-tracing-opentelemetry >= 1.0.0b11",
     "azure-search-documents >= 11.6.0b4",
     "azure-cosmos ~= 4.7"

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_as_kernel_function.py

@@ -71,10 +71,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         # Create the agent definition
         agent_definition = await client.agents.create_agent(

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_azure_ai_search.py

@@ -3,7 +3,8 @@
 import asyncio
 import logging
 
-from azure.ai.projects.models import AzureAISearchTool, ConnectionType
+from azure.ai.agents.models import AzureAISearchTool
+from azure.ai.projects.models import ConnectionType
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings, AzureAIAgentThread
@@ -39,17 +40,12 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
-        conn_list = await client.connections.list()
-
         ai_search_conn_id = ""
-        for conn in conn_list:
-            if conn.connection_type == ConnectionType.AZURE_AI_SEARCH and conn.authentication_type == "ApiKey":
-                ai_search_conn_id = conn.id
+        async for connection in client.connections.list():
+            if connection.type == ConnectionType.AZURE_AI_SEARCH:
+                ai_search_conn_id = connection.id
                 break
 
         ai_search = AzureAISearchTool(index_connection_id=ai_search_conn_id, index_name=AZURE_AI_SEARCH_INDEX_NAME)

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_bing_grounding.py

@@ -2,7 +2,7 @@
 
 import asyncio
 
-from azure.ai.projects.models import BingGroundingTool
+from azure.ai.agents.models import BingGroundingTool
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings, AzureAIAgentThread

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_bing_grounding_streaming_with_message_callback.py

@@ -3,7 +3,7 @@
 import asyncio
 from functools import reduce
 
-from azure.ai.projects.models import BingGroundingTool
+from azure.ai.agents.models import BingGroundingTool
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings, AzureAIAgentThread

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_code_interpreter_streaming_with_message_callback.py

@@ -3,7 +3,7 @@
 import asyncio
 from functools import reduce
 
-from azure.ai.projects.models import CodeInterpreterTool
+from azure.ai.agents.models import CodeInterpreterTool
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings, AzureAIAgentThread

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_declarative_code_interpreter.py

@@ -3,7 +3,7 @@
 import asyncio
 import os
 
-from azure.ai.projects.models import FilePurpose
+from azure.ai.agents.models import FilePurpose
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings
@@ -28,15 +28,15 @@
 model:
   id: ${AzureAI:ChatModelId}
   connection:
-    connection_string: ${AzureAI:ConnectionString}
+    endpoint: ${AzureAI:Endpoint}
 tools:
   - type: code_interpreter
     options:
       file_ids:
         - ${AzureAI:FileId1}
 """
 
-settings = AzureAIAgentSettings()  # ChatModelId & ConnectionString come from env vars
+settings = AzureAIAgentSettings()  # ChatModelId & Endpoint come from env vars
 
 
 async def main():
@@ -54,7 +54,7 @@ async def main():
 
         try:
             # Upload the CSV file to the agent service
-            file = await client.agents.upload_file_and_poll(file_path=csv_file_path, purpose=FilePurpose.AGENTS)
+            file = await client.agents.files.upload_and_poll(file_path=csv_file_path, purpose=FilePurpose.AGENTS)
 
             # Create the AzureAI Agent from the YAML spec
             # Note: the extras can be provided in the short-format (shown below) or
@@ -100,7 +100,7 @@ async def main():
         finally:
             # Cleanup: Delete the thread and agent
             await client.agents.delete_agent(agent.id)
-            await client.agents.delete_file(file.id)
+            await client.agents.files.delete(file.id)
 
         """
         Sample output:

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_declarative_file_search.py

@@ -3,7 +3,7 @@
 import asyncio
 import os
 
-from azure.ai.projects.models import OpenAIFile, VectorStore
+from azure.ai.agents.models import VectorStore
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings
@@ -28,15 +28,15 @@
 model:
   id: ${AzureAI:ChatModelId}
   connection:
-    connection_string: ${AzureAI:ConnectionString}
+    endpoint: ${AzureAI:Endpoint}
 tools:
   - type: file_search
     options:
       vector_store_ids:
         - ${AzureAI:VectorStoreId}
 """
 
-settings = AzureAIAgentSettings()  # ChatModelId & ConnectionString come from .env/env vars
+settings = AzureAIAgentSettings()  # ChatModelId & Endpoint come from .env/env vars
 
 
 async def main():
@@ -52,10 +52,8 @@ async def main():
             "employees.pdf",
         )
         # Upload the pdf file to the agent service
-        file: OpenAIFile = await client.agents.upload_file_and_poll(file_path=pdf_file_path, purpose="assistants")
-        vector_store: VectorStore = await client.agents.create_vector_store_and_poll(
-            file_ids=[file.id], name="my_vectorstore"
-        )
+        file = await client.agents.files.upload_and_poll(file_path=pdf_file_path, purpose="assistants")
+        vector_store: VectorStore = await client.agents.vector_stores.create(file_ids=[file.id], name="my_vectorstore")
 
         try:
             # Create the AzureAI Agent from the YAML spec
@@ -82,8 +80,8 @@ async def main():
         finally:
             # Cleanup: Delete the agent, vector store, and file
             await client.agents.delete_agent(agent.id)
-            await client.agents.delete_vector_store(vector_store.id)
-            await client.agents.delete_file(file.id)
+            await client.agents.vector_stores.delete(vector_store.id)
+            await client.agents.files.delete(file.id)
 
         """
         Sample output:

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_declarative_function_calling_from_file.py

@@ -35,11 +35,6 @@ def get_item_price(
         return "$9.99"
 
 
-# Function spec
-
-settings = AzureAIAgentSettings()  # The Spec's ChatModelId & ConnectionString come from .env/env vars
-
-
 async def main():
     async with (
         DefaultAzureCredential() as creds,
@@ -59,7 +54,7 @@ async def main():
                 file_path,
                 plugins=[MenuPlugin()],
                 client=client,
-                settings=settings,
+                settings=AzureAIAgentSettings(),  # The Spec's ChatModelId & Endpoint come from .env/env vars
             )
 
             # Create the agent

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_declarative_openapi.py

@@ -24,7 +24,7 @@
 model:
   id: ${AzureAI:ChatModelId}
   connection:
-    connection_string: ${AzureAI:ConnectionString}
+    endpoint: ${AzureAI:Endpoint}
   options:
     temperature: 0.4
 tools:
@@ -153,7 +153,7 @@
           schemes: {}
 """
 
-settings = AzureAIAgentSettings()  # ChatModelId & ConnectionString come from .env/env vars
+settings = AzureAIAgentSettings()  # ChatModelId & Endpoint come from .env/env vars
 
 
 async def main():

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_file_manipulation.py

@@ -3,7 +3,7 @@
 import asyncio
 import os
 
-from azure.ai.projects.models import CodeInterpreterTool, FilePurpose
+from azure.ai.agents.models import CodeInterpreterTool, FilePurpose
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import AzureAIAgent, AzureAIAgentSettings, AzureAIAgentThread
@@ -22,10 +22,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         csv_file_path = os.path.join(
             os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))),
@@ -34,7 +31,7 @@ async def main() -> None:
             "sales.csv",
         )
 
-        file = await client.agents.upload_file_and_poll(file_path=csv_file_path, purpose=FilePurpose.AGENTS)
+        file = await client.agents.files.upload_and_poll(file_path=csv_file_path, purpose=FilePurpose.AGENTS)
 
         code_interpreter = CodeInterpreterTool(file_ids=[file.id])
 

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_message_callback.py

@@ -60,10 +60,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         AGENT_NAME = "Host"
         AGENT_INSTRUCTIONS = "Answer questions about the menu."

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_message_callback_streaming.py

@@ -61,10 +61,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         # Create agent definition
         agent_definition = await client.agents.create_agent(

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_prompt_templating.py

@@ -56,10 +56,7 @@ async def invoke_agent_with_template(template_str: str, template_format: str, de
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         # Create agent definition
         agent_definition = await client.agents.create_agent(

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_streaming.py

@@ -40,10 +40,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         AGENT_NAME = "Host"
         AGENT_INSTRUCTIONS = "Answer questions about the menu."

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_structured_outputs.py

@@ -3,7 +3,7 @@
 import asyncio
 from enum import Enum
 
-from azure.ai.projects.models import (
+from azure.ai.agents.models import (
     ResponseFormatJsonSchema,
     ResponseFormatJsonSchemaType,
 )
@@ -38,10 +38,7 @@ async def main():
     ai_agent_settings = AzureAIAgentSettings()
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         # Create the agent definition
         agent_definition = await client.agents.create_agent(

python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_truncation_strategy.py

@@ -2,7 +2,7 @@
 
 import asyncio
 
-from azure.ai.projects.models import TruncationObject
+from azure.ai.agents.models import TruncationObject
 from azure.identity.aio import DefaultAzureCredential
 
 from semantic_kernel.agents import (
@@ -28,10 +28,7 @@ async def main() -> None:
 
     async with (
         DefaultAzureCredential() as creds,
-        AzureAIAgent.create_client(
-            credential=creds,
-            conn_str=ai_agent_settings.project_connection_string.get_secret_value(),
-        ) as client,
+        AzureAIAgent.create_client(credential=creds, endpoint=ai_agent_settings.endpoint) as client,
     ):
         # Create the agent definition
         agent_definition = await client.agents.create_agent(

python/samples/concepts/resources/declarative_spec/spec.yaml

@@ -5,7 +5,7 @@ instructions: Use the provided functions to answer questions about the menu.
 model:
   id: ${AzureAI:ChatModelId}
   connection:
-    connection_string: ${AzureAI:ConnectionString}
+    endpoint: ${AzureAI:Endpoint}
   options:
     temperature: 0.4
 tools: