Update eval samples (#134)

shyamnamboodiripad · web-flow · commit 3bb2efb4f898 · 2025-09-03T07:34:40.000-04:00
diff --git a/src/microsoft-extensions-ai-evaluation/api/.config/dotnet-tools.json b/src/microsoft-extensions-ai-evaluation/api/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "microsoft.extensions.ai.evaluation.console": {
-      "version": "9.7.0",
+      "version": "9.8.0",
       "commands": [
         "aieval"
       ],
diff --git a/src/microsoft-extensions-ai-evaluation/api/INSTRUCTIONS.md b/src/microsoft-extensions-ai-evaluation/api/INSTRUCTIONS.md
@@ -128,21 +128,22 @@ AI Foundry Evaluation service and require some special setup. The corresponding
 be enabled by setting the following environment variables.
 
 1. Firstly, you need an [Azure subscription](https://azure.microsoft.com/).
-2. Within this subscription, create a
-   [resource group](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)
-   within one of the
-   [Azure regions that support Azure AI Foundry Evaluation service](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/develop/evaluate-sdk#region-support).
-3. Next create an [Azure AI hub](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/ai-resources) within the
-   same resource group and region.
-4. Finally, create an [Azure AI project](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects?tabs=ai-studio)
-   within this hub.
-5. Once you have created the above artifacts, set the following environment variables to configure the examples to use
-   the Azure AI Foundry Evaluation service from the above AI project.
+2. Within this subscription, create an [Azure AI Foundry project](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects?tabs=ai-foundry&pivots=fdp-project).
+3. Finally, copy the endpoint URL for the above Azure AI Foundry project and set the following environment variable.
+   This will configure the examples to use the Azure AI Foundry Evaluation service from the above Azure AI Foundry
+   project.
+
+```
+set EVAL_SAMPLE_AZURE_AI_PROJECT_ENDPOINT=<The endpoint URL for the above Azure AI Foundry project>
+```
+
+**Note:** If you are using an older [hub-based Azure AI Foundry project](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects?tabs=ai-foundry&pivots=hub-project),
+set the following environment variables instead.
 
 ```
-set EVAL_SAMPLE_AZURE_SUBSCRIPTION_ID=<The ID of the above Azure subscription>
-set EVAL_SAMPLE_AZURE_RESOURCE_GROUP=<The name of the above Azure resource group>
-set EVAL_SAMPLE_AZURE_AI_PROJECT=<The name of the above Azure AI project>
+set EVAL_SAMPLE_AZURE_SUBSCRIPTION_ID=<The ID of the Azure subscription containing your hub-based Azure AI Foundry project>
+set EVAL_SAMPLE_AZURE_RESOURCE_GROUP=<The name of the Azure resource group containing your hub-based Azure AI Foundry project>
+set EVAL_SAMPLE_AZURE_AI_PROJECT=<The name of the your hub-based Azure AI Foundry project>
 ```
 
 ## Running Azure storage examples
@@ -164,7 +165,7 @@ variables.
    the storage container created above.
 
 ```
-set EVAL_SAMPLE_AZURE_STORAGE_ACCOUNT_ENDPOINT=<The endpoint url of the above Azure storage account>
+set EVAL_SAMPLE_AZURE_STORAGE_ACCOUNT_ENDPOINT=<The endpoint URL of the above Azure storage account>
 set EVAL_SAMPLE_AZURE_STORAGE_CONTAINER=<The name of the above Azure storage container>
 ```
 
diff --git a/src/microsoft-extensions-ai-evaluation/api/evaluation/Evaluation.csproj b/src/microsoft-extensions-ai-evaluation/api/evaluation/Evaluation.csproj
@@ -11,18 +11,18 @@
 
   <ItemGroup>
     <PackageReference Include="Azure.AI.OpenAI" Version="2.1.0" />
-    <PackageReference Include="Azure.Identity" Version="1.14.2" />
+    <PackageReference Include="Azure.Identity" Version="1.15.0" />
     <PackageReference Include="AwesomeAssertions" Version="9.1.0" />
-    <PackageReference Include="Microsoft.Extensions.AI" Version="9.7.1" />
-    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.7.1" />
-    <PackageReference Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.7.1-preview.1.25365.4" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.7.1-preview.1.25365.4" />
+    <PackageReference Include="Microsoft.Extensions.AI" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.8.0-preview.1.25412.6" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.8.0-preview.1.25412.6" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="3.9.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="3.9.3" />
-    <PackageReference Include="OllamaSharp" Version="5.3.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.10.4" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.10.4" />
+    <PackageReference Include="OllamaSharp" Version="5.3.6" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example04_InvokingCustomEvaluators.cs b/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example04_InvokingCustomEvaluators.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
 using Microsoft.Extensions.AI.Evaluation;
 
 namespace Evaluation;
diff --git a/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example05_AttachingDiagnosticsToMetrics.cs b/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example05_AttachingDiagnosticsToMetrics.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
 using Microsoft.Extensions.AI.Evaluation;
 
 namespace Evaluation;
diff --git a/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example06_ChangingInterpretationOfMetrics.cs b/src/microsoft-extensions-ai-evaluation/api/evaluation/EvaluationExamples.Example06_ChangingInterpretationOfMetrics.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
 using Microsoft.Extensions.AI.Evaluation;
 
 namespace Evaluation;
diff --git a/src/microsoft-extensions-ai-evaluation/api/evaluation/Setup/EnvironmentVariables.cs b/src/microsoft-extensions-ai-evaluation/api/evaluation/Setup/EnvironmentVariables.cs
@@ -75,6 +75,12 @@ public static string OpenAIModel
     #endregion
 
     #region Azure AI Foundry (for content safety evaluation examples)
+    public const string AzureAIProjectEndpointVariableName = "EVAL_SAMPLE_AZURE_AI_PROJECT_ENDPOINT";
+    public static string? AzureAIProjectEndpoint =>
+        TryGetEnvironmentVariable(AzureAIProjectEndpointVariableName, out string? value)
+            ? value
+            : null;
+
     public const string AzureSubscriptionIdVariableName = "EVAL_SAMPLE_AZURE_SUBSCRIPTION_ID";
     public static string? AzureSubscriptionId =>
         TryGetEnvironmentVariable(AzureSubscriptionIdVariableName, out string? value)
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/Reporting.csproj b/src/microsoft-extensions-ai-evaluation/api/reporting/Reporting.csproj
@@ -11,22 +11,22 @@
 
   <ItemGroup>
     <PackageReference Include="Azure.AI.OpenAI" Version="2.1.0" />
-    <PackageReference Include="Azure.Identity" Version="1.14.2" />
+    <PackageReference Include="Azure.Identity" Version="1.15.0" />
     <PackageReference Include="AwesomeAssertions" Version="9.1.0" />
-    <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.7" />
-    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.7.1" />
-    <PackageReference Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.7.1-preview.1.25365.4" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.NLP" Version="9.7.0-preview.1.25356.2" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Reporting" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Reporting.Azure" Version="9.7.0" />
-    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="9.7.0-preview.1.25356.2" />
-    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.7.1-preview.1.25365.4" />
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.8" />
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.8.0-preview.1.25412.6" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.NLP" Version="9.8.0-preview.1.25412.6" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Reporting" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Reporting.Azure" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="9.8.0-preview.1.25412.6" />
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.8.0-preview.1.25412.6" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="3.9.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="3.9.3" />
-    <PackageReference Include="OllamaSharp" Version="5.3.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.10.4" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.10.4" />
+    <PackageReference Include="OllamaSharp" Version="5.3.6" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example05_InvokingEvaluatorsThatNeedAdditionalContext.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example05_InvokingEvaluatorsThatNeedAdditionalContext.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Setup;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Setup;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Quality;
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example06_AttachingDiagnosticsToMetrics.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example06_AttachingDiagnosticsToMetrics.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example07_ChangingInterpretationOfMetrics.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example07_ChangingInterpretationOfMetrics.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example08_RunningSafetyEvaluators.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example08_RunningSafetyEvaluators.cs
@@ -3,15 +3,14 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Diagnostics.CodeAnalysis;
-using Azure.Identity;
-using Evaluation.Setup;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Azure.Identity;
+using Evaluation.Setup;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
 using Microsoft.Extensions.AI.Evaluation.Safety;
-using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
 namespace Reporting;
 
@@ -23,15 +22,34 @@ public partial class ReportingExamples
     /// need in order to communicate with the Azure AI Foundry Evaluation service.
 
     private static readonly ContentSafetyServiceConfiguration? s_safetyServiceConfiguration =
-        EnvironmentVariables.AzureSubscriptionId is null ||
-        EnvironmentVariables.AzureResourceGroup is null ||
-        EnvironmentVariables.AzureAIProject is null
-            ? null
-            : new ContentSafetyServiceConfiguration(
+        GetSafetyServiceConfiguration();
+
+    private static ContentSafetyServiceConfiguration? GetSafetyServiceConfiguration()
+    {
+        if (EnvironmentVariables.AzureAIProjectEndpoint is null)
+        {
+            if (EnvironmentVariables.AzureSubscriptionId is null ||
+                EnvironmentVariables.AzureResourceGroup is null ||
+                EnvironmentVariables.AzureAIProject is null)
+            {
+                return null;
+            }
+            else
+            {
+                return new ContentSafetyServiceConfiguration(
+                    credential: new DefaultAzureCredential(),
+                    subscriptionId: EnvironmentVariables.AzureSubscriptionId,
+                    resourceGroupName: EnvironmentVariables.AzureResourceGroup,
+                    projectName: EnvironmentVariables.AzureAIProject);
+            }
+        }
+        else
+        {
+            return new ContentSafetyServiceConfiguration(
                 credential: new DefaultAzureCredential(),
-                subscriptionId: EnvironmentVariables.AzureSubscriptionId,
-                resourceGroupName: EnvironmentVariables.AzureResourceGroup,
-                projectName: EnvironmentVariables.AzureAIProject);
+                endpoint: new Uri(EnvironmentVariables.AzureAIProjectEndpoint));
+        }
+    }
 
     /// The <see cref="ContentSafetyServiceConfiguration"/> configured above can then be converted to a
     /// <see cref="ChatConfiguration"/> by calling
@@ -156,10 +174,13 @@ private static void SkipTestIfSafetyEvaluatorsAreNotConfigured()
         {
             Assert.Inconclusive(
                 $"""
-                The test was skipped since the following environment variables were not set. Set these variables to configure the Azure AI Foundry Evaluation service for use in this test:
-                set {EnvironmentVariables.AzureSubscriptionIdVariableName}=<The subscription ID of the Azure account that contains the below Azure AI project>
-                set {EnvironmentVariables.AzureResourceGroupVariableName}=<The name of the Azure resource group under the above subscription that contains the below Azure AI project>
-                set {EnvironmentVariables.AzureAIProjectVariableName}=<The name of the Azure AI project under which the content safety evaluations are to be executed>
+                The test was skipped since the following environment variable was not set. Set this variable to configure the Azure AI Foundry Evaluation service for use in this test:
+                set {EnvironmentVariables.AzureAIProjectEndpointVariableName}=<The endpoint URL of the Azure AI Foundry project under which the content safety evaluations are to be executed>
+                
+                Note: If you are using an older hub-based Azure AI Foundry project, set the following three environment variables instead:
+                set {EnvironmentVariables.AzureSubscriptionIdVariableName}=<The subscription ID of the Azure account that contains the below hub-based Azure AI Foundry project>
+                set {EnvironmentVariables.AzureResourceGroupVariableName}=<The name of the Azure resource group under the above subscription that contains the below hub-based Azure AI Foundry project>
+                set {EnvironmentVariables.AzureAIProjectVariableName}=<The name of the hub-based Azure AI Foundry project under which the content safety evaluations are to be executed>
                 """);
         }
 
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example10_RunningQualityAndSafetyEvaluatorsTogether.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example10_RunningQualityAndSafetyEvaluatorsTogether.cs
@@ -2,10 +2,10 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Evaluation.Evaluators;
-using Evaluation.Setup;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
+using Evaluation.Setup;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Quality;
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example13_UsingAzureStorage_01.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.Example13_UsingAzureStorage_01.cs
@@ -3,10 +3,10 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Diagnostics.CodeAnalysis;
+using AwesomeAssertions;
 using Azure.Identity;
 using Azure.Storage.Files.DataLake;
 using Evaluation.Setup;
-using AwesomeAssertions;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
@@ -72,7 +72,7 @@ private static void SkipTestIfAzureStorageNotConfigured()
             Assert.Inconclusive(
                 $"""
                 The test was skipped since the following environment variables were not set. Set these variables to configure the Azure storage for use in this test:
-                set {EnvironmentVariables.AzureStorageAccountEndpoint}=<The endpoint url that identifies the Azure Data Lake Gen2 enabled storage account that contains the below storage container>
+                set {EnvironmentVariables.AzureStorageAccountEndpoint}=<The endpoint URL that identifies the Azure Data Lake Gen2 enabled storage account that contains the below storage container>
                 set {EnvironmentVariables.AzureStorageContainer}=<The name of the Azure storage container>
                 """);
         }
diff --git a/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.cs b/src/microsoft-extensions-ai-evaluation/api/reporting/ReportingExamples.cs
@@ -3,10 +3,10 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Reflection;
-using Evaluation.Evaluators;
-using Evaluation.Setup;
 using AwesomeAssertions;
 using AwesomeAssertions.Execution;
+using Evaluation.Evaluators;
+using Evaluation.Setup;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Evaluation;
 using Microsoft.Extensions.AI.Evaluation.Quality;
