From a69807238ebc05958380ab6c875ce7e394e50f76 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Thu, 9 Oct 2025 12:19:37 -0400 Subject: [PATCH 01/16] Initial change for adding consolidated mode --- .../Discovery/BaseDiscoveryStrategy.cs | 10 +- .../Discovery/CommandGroupServerProvider.cs | 3 +- .../ConsolidatedToolDiscoveryStrategy.cs | 83 +++ .../Commands/Discovery/McpServerMetadata.cs | 3 + .../Commands/ServiceCollectionExtensions.cs | 19 + .../Server/Commands/ServiceStartCommand.cs | 5 +- .../Commands/ToolLoading/ServerToolLoader.cs | 22 +- .../Models/ConsolidatedToolDefinition.cs | 32 ++ .../src/Areas/Server/Options/ModeTypes.cs | 5 + .../src/Commands/BaseCommand.cs | 1 + .../src/Commands/CommandGroup.cs | 1 + .../src/Commands/IBaseCommand.cs | 7 + .../Areas/Server/ServerCommandTests.cs | 2 +- .../Areas/Server/CommandFactoryHelpers.cs | 71 ++- .../ConsolidatedToolDiscoveryStrategyTests.cs | 532 ++++++++++++++++++ .../src/AzureBestPracticesSetup.cs | 2 +- .../src/Commands/BestPracticesCommand.cs | 2 + .../AzureTerraformBestPracticesGetCommand.cs | 2 + .../src/Commands/BicepSchemaGetCommand.cs | 2 + 19 files changed, 788 insertions(+), 16 deletions(-) create mode 100644 core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs create mode 100644 core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs create mode 100644 core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs index 87c22f0d7b..6e93a76ef6 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs @@ -72,13 +72,13 @@ public async Task GetOrCreateClientAsync(string name, McpClientOptio throw new ArgumentNullException(nameof(name), "Server name cannot be null or empty."); } - if (_clientCache.TryGetValue(name, out var client)) - { - return client; - } + // if (_clientCache.TryGetValue(name, out var client)) + // { + // return client; + // } var serverProvider = await FindServerProviderAsync(name); - client = await serverProvider.CreateClientAsync(clientOptions ?? new McpClientOptions()); + var client = await serverProvider.CreateClientAsync(clientOptions ?? new McpClientOptions()); _clientCache[name] = client; return client; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs index 93e03c7cf8..9af8d16801 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs @@ -80,7 +80,8 @@ public McpServerMetadata CreateMetadata() { Id = _commandGroup.Name, Name = _commandGroup.Name, - Description = _commandGroup.Description + Description = _commandGroup.Description, + ToolMetadata = _commandGroup.ToolMetadata, }; } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs new file mode 100644 index 0000000000..98d98e6efa --- /dev/null +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Mcp.Core.Areas.Server.Options; +using Azure.Mcp.Core.Commands; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Mcp.Core.Areas.Server.Commands.Discovery; + +/// +/// Discovery strategy that exposes command groups as MCP servers. +/// This strategy converts Azure CLI command groups into MCP servers, allowing them to be accessed via the MCP protocol. +/// +/// The command factory used to access available command groups. +/// Options for configuring the service behavior. +/// Logger instance for this discovery strategy. +public sealed class ConsolidatedToolDiscoveryStrategy(CommandFactory commandFactory, IOptions options, ILogger logger) : BaseDiscoveryStrategy(logger) +{ + private readonly CommandFactory _commandFactory = commandFactory; + private readonly IOptions _options = options; + + /// + /// Gets or sets the entry point to use for the command group servers. + /// This can be used to specify a custom entry point for the commands. + /// + public string? EntryPoint { get; set; } = null; + + /// + /// Discovers available command groups and converts them to MCP server providers. + /// + /// A collection of command group server providers. + public override Task> DiscoverServersAsync() + { + // Find all commands with the same CompositeToolMapped value + var allCommands = _commandFactory.AllCommands; + var matchingCommands = allCommands + .Where(kvp => !string.IsNullOrWhiteSpace(kvp.Value.CompositeToolMapped) && + string.Equals(kvp.Value.CompositeToolMapped, "get_azure_best_practices", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + // Create a new CommandGroup and add the matching commands + var commandGroup = new CommandGroup("get_azure_best_practices", "Retrieve Azure best practices and infrastructure schema for code generation, deployment, and operations. Covers general Azure practices, Azure Functions best practices, Terraform configurations, Bicep template schemas, and deployment best practices."); + _commandFactory.RootGroup.AddSubGroup(commandGroup); + foreach (var (commandName, command) in matchingCommands) + { + commandGroup.AddCommand(commandName, command); + // Extract just the command name (remove any prefix) + // TODO: It is going to be all get commands??? + // var simpleName = commandName.Replace("azmcp_", "").Replace("_", "."); + // commandGroup.AddCommand(simpleName, command); + } + commandGroup.ToolMetadata = new ToolMetadata + { + Destructive = false, + Idempotent = true, + OpenWorld = false, + ReadOnly = true, + LocalRequired = false, + Secret = false + }; + + CommandGroupServerProvider serverProvider = new CommandGroupServerProvider(commandGroup) + { + ReadOnly = _options.Value.ReadOnly ?? false, + EntryPoint = EntryPoint, + }; + + var providers = new List { serverProvider }; + // providers.AddRange(_commandFactory.RootGroup.SubGroup + // .Where(group => _options.Value.Namespace == null || + // _options.Value.Namespace.Length == 0 || + // _options.Value.Namespace.Contains(group.Name, StringComparer.OrdinalIgnoreCase)) + // .Select(group => new CommandGroupServerProvider(group) + // { + // ReadOnly = _options.Value.ReadOnly ?? false, + // EntryPoint = EntryPoint, + // }) + // .Cast(); + + return Task.FromResult>(providers); + } +} diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs index ac3f21dd51..d02c557d67 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using ModelContextProtocol.Client; +using Azure.Mcp.Core.Commands; namespace Azure.Mcp.Core.Areas.Server.Commands.Discovery; @@ -27,6 +28,8 @@ public sealed class McpServerMetadata(string id = "", string name = "", string d /// Gets or sets a description of the server's purpose or capabilities. /// public string Description { get; set; } = description; + + public ToolMetadata? ToolMetadata { get; set; } } /// diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceCollectionExtensions.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceCollectionExtensions.cs index 0ac985b92b..56f779fb0f 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceCollectionExtensions.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceCollectionExtensions.cs @@ -78,6 +78,7 @@ public static IServiceCollection AddAzureMcpServer(this IServiceCollection servi services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Register server providers services.AddSingleton(); @@ -101,6 +102,20 @@ public static IServiceCollection AddAzureMcpServer(this IServiceCollection servi return new CompositeDiscoveryStrategy(discoveryStrategies, logger); }); } + else if (serviceStartOptions.Mode == ModeTypes.ConsolidatedProxy) + { + services.AddSingleton(sp => + { + var discoveryStrategies = new List + { + sp.GetRequiredService(), + sp.GetRequiredService(), + }; + + var logger = sp.GetRequiredService>(); + return new CompositeDiscoveryStrategy(discoveryStrategies, logger); + }); + } // Configure tool loading based on mode if (serviceStartOptions.Mode == ModeTypes.SingleToolProxy) @@ -126,6 +141,10 @@ public static IServiceCollection AddAzureMcpServer(this IServiceCollection servi return new CompositeToolLoader(toolLoaders, loggerFactory.CreateLogger()); }); } + else if (serviceStartOptions.Mode == ModeTypes.ConsolidatedProxy) + { + services.AddSingleton(); + } else if (serviceStartOptions.Mode == ModeTypes.All) { services.AddSingleton(); diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceStartCommand.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceStartCommand.cs index 42cc56b4de..91b2f1b1e9 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceStartCommand.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ServiceStartCommand.cs @@ -85,7 +85,7 @@ public override async Task ExecuteAsync(CommandContext context, if (!IsValidMode(mode)) { - throw new ArgumentException($"Invalid mode '{mode}'. Valid modes are: {ModeTypes.SingleToolProxy}, {ModeTypes.NamespaceProxy}, {ModeTypes.All}."); + throw new ArgumentException($"Invalid mode '{mode}'. Valid modes are: {ModeTypes.SingleToolProxy}, {ModeTypes.NamespaceProxy}, {ModeTypes.All}, {ModeTypes.ConsolidatedProxy}."); } var enableInsecureTransports = parseResult.GetValueOrDefault(_enableInsecureTransportsOption); @@ -125,7 +125,8 @@ private static bool IsValidMode(string? mode) { return mode == ModeTypes.SingleToolProxy || mode == ModeTypes.NamespaceProxy || - mode == ModeTypes.All; + mode == ModeTypes.All || + mode == ModeTypes.ConsolidatedProxy; } /// diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs index e1c1f07278..83ed2661b9 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs @@ -82,6 +82,18 @@ Sub commands are routed to MCP servers that require specific fields inside the " InputSchema = ToolSchema, }; + if (metadata.ToolMetadata != null) + { + tool.Annotations = new ToolAnnotations + { + DestructiveHint = metadata.ToolMetadata.Destructive, + IdempotentHint = metadata.ToolMetadata.Idempotent, + OpenWorldHint = metadata.ToolMetadata.OpenWorld, + ReadOnlyHint = metadata.ToolMetadata.ReadOnly, + // Secret and LocalRequired are not annotated as hints + }; + } + allToolsResponse.Tools.Add(tool); } @@ -342,10 +354,10 @@ private async Task InvokeToolLearn(RequestContext private async Task> GetChildToolListAsync(RequestContext request, string tool) { - if (_cachedToolLists.TryGetValue(tool, out var cachedList)) - { - return cachedList; - } + // if (_cachedToolLists.TryGetValue(tool, out var cachedList)) + // { + // return cachedList; + // } if (string.IsNullOrWhiteSpace(request.Params?.Name)) { @@ -369,6 +381,8 @@ private async Task> GetChildToolListAsync(RequestContext t.ProtocolTool) .Where(t => !(options?.Value?.ReadOnly ?? false) || (t.Annotations?.ReadOnlyHint == true)) + // SecretHint doesn't exist yet + // .Where(t => !(options?.Value?.Secret ?? false) || (t.Annotations?.SecretHint == true)) .ToList(); _cachedToolLists[tool] = list; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs new file mode 100644 index 0000000000..57a3264d84 --- /dev/null +++ b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Azure.Mcp.Core.Commands; + +namespace Azure.Mcp.Core.Areas.Server.Models; + +/// +/// Represents a composite tool definition that groups multiple related Azure operations together. +/// Used to create consolidated tools from the azure_mcp_consolidated_tools JSON configuration. +/// +public sealed class ConsolidatedToolDefinition +{ + /// + /// Gets or sets the name of the composite tool. + /// + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the description of the composite tool's capabilities and purpose. + /// + [JsonPropertyName("description")] + public string Description { get; set; } = string.Empty; + + /// + /// Gets or sets the tool metadata containing capability information. + /// + [JsonPropertyName("toolMetadata")] + public ToolMetadata? ToolMetadata { get; set; } +} \ No newline at end of file diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Options/ModeTypes.cs b/core/Azure.Mcp.Core/src/Areas/Server/Options/ModeTypes.cs index 792a1cd373..6fd9355a24 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Options/ModeTypes.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Options/ModeTypes.cs @@ -23,4 +23,9 @@ internal static class ModeTypes /// All tools mode - exposes all Azure MCP tools individually (one tool per command). /// public const string All = "all"; + + /// + /// Consolidated tools mode - exposes consolidated tools that group related Azure operations together. + /// + public const string ConsolidatedProxy = "consolidated"; } diff --git a/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs b/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs index 634b498b0d..bb2f0e923f 100644 --- a/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs +++ b/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs @@ -33,6 +33,7 @@ protected BaseCommand() public abstract string Description { get; } public abstract string Title { get; } public abstract ToolMetadata Metadata { get; } + public virtual string? CompositeToolMapped { get; protected set; } protected virtual void RegisterOptions(Command command) { diff --git a/core/Azure.Mcp.Core/src/Commands/CommandGroup.cs b/core/Azure.Mcp.Core/src/Commands/CommandGroup.cs index 0e323d343f..d5df6ea22e 100644 --- a/core/Azure.Mcp.Core/src/Commands/CommandGroup.cs +++ b/core/Azure.Mcp.Core/src/Commands/CommandGroup.cs @@ -10,6 +10,7 @@ public class CommandGroup(string name, string description) public List SubGroup { get; } = []; public Dictionary Commands { get; } = []; public Command Command { get; } = new Command(name, description); + public ToolMetadata? ToolMetadata { get; set; } public void AddCommand(string path, IBaseCommand command) { diff --git a/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs b/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs index 42d6bfa9bd..efc44d9213 100644 --- a/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs +++ b/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs @@ -33,6 +33,13 @@ public interface IBaseCommand /// ToolMetadata Metadata { get; } + /// + /// Gets the name of the composite tool that this command maps to. + /// Used for discovery and routing in the consolidated tool system. + /// If null, the command is not mapped to any composite tool. + /// + string? CompositeToolMapped { get; } + /// /// Gets the command definition /// diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs index 14e35734e3..0b19a1ca43 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs @@ -124,7 +124,7 @@ public async Task AllMode_LoadsAllIndividualTools() public async Task SingleProxyMode_LoadsSingleAzureTool() { // Arrange - await using var client = await CreateClientAsync("server", "start", "--mode", "single"); + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated"); // Act var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/CommandFactoryHelpers.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/CommandFactoryHelpers.cs index 6b85f9cb15..47cd7d2ea7 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/CommandFactoryHelpers.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/CommandFactoryHelpers.cs @@ -3,13 +3,46 @@ using System.Diagnostics; using Azure.Mcp.Core.Areas; +using Azure.Mcp.Core.Areas.Group; +using Azure.Mcp.Core.Areas.Server; using Azure.Mcp.Core.Areas.Subscription; +using Azure.Mcp.Core.Areas.Tools; using Azure.Mcp.Core.Commands; using Azure.Mcp.Core.Services.Telemetry; +using Azure.Mcp.Tools.Acr; +using Azure.Mcp.Tools.Aks; using Azure.Mcp.Tools.AppConfig; +using Azure.Mcp.Tools.AppLens; +using Azure.Mcp.Tools.Authorization; +using Azure.Mcp.Tools.AzureBestPractices; +using Azure.Mcp.Tools.AzureIsv; +using Azure.Mcp.Tools.AzureManagedLustre; +using Azure.Mcp.Tools.AzureTerraformBestPractices; +using Azure.Mcp.Tools.BicepSchema; +using Azure.Mcp.Tools.CloudArchitect; +using Azure.Mcp.Tools.Cosmos; using Azure.Mcp.Tools.Deploy; +using Azure.Mcp.Tools.EventGrid; +using Azure.Mcp.Tools.Extension; +using Azure.Mcp.Tools.Foundry; +using Azure.Mcp.Tools.FunctionApp; +using Azure.Mcp.Tools.Grafana; using Azure.Mcp.Tools.KeyVault; +using Azure.Mcp.Tools.Kusto; +using Azure.Mcp.Tools.LoadTesting; +using Azure.Mcp.Tools.Marketplace; +using Azure.Mcp.Tools.Monitor; +using Azure.Mcp.Tools.MySql; +using Azure.Mcp.Tools.Postgres; +using Azure.Mcp.Tools.Quota; +using Azure.Mcp.Tools.Redis; +using Azure.Mcp.Tools.ResourceHealth; +using Azure.Mcp.Tools.Search; +using Azure.Mcp.Tools.ServiceBus; +using Azure.Mcp.Tools.Sql; using Azure.Mcp.Tools.Storage; +using Azure.Mcp.Tools.VirtualDesktop; +using Azure.Mcp.Tools.Workbooks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using ModelContextProtocol.Protocol; @@ -28,11 +61,45 @@ public static CommandFactory CreateCommandFactory(IServiceProvider? serviceProvi var telemetryService = services.GetService() ?? new NoOpTelemetryService(); IAreaSetup[] areaSetups = [ + // Core areas new SubscriptionSetup(), + new GroupSetup(), + + // Tool areas + new AcrSetup(), + new AksSetup(), + new AppConfigSetup(), + new AppLensSetup(), + new AuthorizationSetup(), + new AzureBestPracticesSetup(), + new AzureIsvSetup(), + new AzureManagedLustreSetup(), + new AzureTerraformBestPracticesSetup(), + new BicepSchemaSetup(), + new CloudArchitectSetup(), + new CosmosSetup(), + new DeploySetup(), + new EventGridSetup(), + new ExtensionSetup(), + new FoundrySetup(), + new FunctionAppSetup(), + new GrafanaSetup(), new KeyVaultSetup(), + new KustoSetup(), + new LoadTestingSetup(), + new MarketplaceSetup(), + new MonitorSetup(), + new MySqlSetup(), + new PostgresSetup(), + new QuotaSetup(), + new RedisSetup(), + new ResourceHealthSetup(), + new SearchSetup(), + new ServiceBusSetup(), + new SqlSetup(), new StorageSetup(), - new DeploySetup(), - new AppConfigSetup() + new VirtualDesktopSetup(), + new WorkbooksSetup() ]; var commandFactory = new CommandFactory(services, areaSetups, telemetryService, logger); diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs new file mode 100644 index 0000000000..8c2a94b568 --- /dev/null +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs @@ -0,0 +1,532 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Mcp.Core.Areas.Server.Commands.Discovery; +using Azure.Mcp.Core.Areas.Server.Options; +using Azure.Mcp.Core.Commands; +using Azure.Mcp.Tests.Client.Helpers; +using Xunit; + +namespace Azure.Mcp.Core.UnitTests.Areas.Server.Commands.Discovery; + +public class ConsolidatedToolDiscoveryStrategyTests +{ + private static ConsolidatedToolDiscoveryStrategy CreateStrategy( + CommandFactory? commandFactory = null, + ServiceStartOptions? options = null, + string? entryPoint = null) + { + var factory = commandFactory ?? CommandFactoryHelpers.CreateCommandFactory(); + var startOptions = Microsoft.Extensions.Options.Options.Create(options ?? new ServiceStartOptions()); + var logger = NSubstitute.Substitute.For>(); + var strategy = new ConsolidatedToolDiscoveryStrategy(factory, startOptions, logger); + if (entryPoint != null) + { + strategy.EntryPoint = entryPoint; + } + return strategy; + } + + [Fact] + public void Constructor_WithNullCommandFactory_DoesNotThrow() + { + // Arrange + var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); + var logger = NSubstitute.Substitute.For>(); + + // Act & Assert + // Primary constructor syntax doesn't automatically validate null parameters + var strategy = new ConsolidatedToolDiscoveryStrategy(null!, options, logger); + Assert.NotNull(strategy); + } + + [Fact] + public void Constructor_WithNullOptions_DoesNotThrow() + { + // Arrange + var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); + var logger = NSubstitute.Substitute.For>(); + + // Act & Assert + // Primary constructor syntax doesn't automatically validate null parameters + var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, null!, logger); + Assert.NotNull(strategy); + } + + [Fact] + public void Constructor_WithValidParameters_InitializesCorrectly() + { + // Arrange + var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); + var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); + var logger = NSubstitute.Substitute.For>(); + + // Act + var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger); + + // Assert + Assert.NotNull(strategy); + Assert.Null(strategy.EntryPoint); // Default should be null + } + + [Fact] + public void EntryPoint_DefaultsToNull() + { + // Arrange + var strategy = CreateStrategy(); + + // Act & Assert + Assert.Null(strategy.EntryPoint); + } + + [Fact] + public void EntryPoint_CanBeSetAndRetrieved() + { + // Arrange + var strategy = CreateStrategy(); + var azmcpPath = McpTestUtilities.GetAzMcpExecutablePath(); + + // Act + strategy.EntryPoint = azmcpPath; + + // Assert + Assert.Equal(azmcpPath, strategy.EntryPoint); + } + + [Fact] + public void EntryPoint_WhenSetToEmpty_RemainsEmpty() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + strategy.EntryPoint = ""; + + // Assert + // The strategy itself just stores the value as-is + // The defaulting behavior happens in CommandGroupServerProvider + Assert.Equal("", strategy.EntryPoint); + } + + [Fact] + public void EntryPoint_WhenSetToWhitespace_RemainsWhitespace() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + strategy.EntryPoint = " "; + + // Assert + // The strategy itself just stores the value as-is + // The defaulting behavior happens in CommandGroupServerProvider + Assert.Equal(" ", strategy.EntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_WithDefaultOptions_ReturnsConsolidatedToolProvider() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + Assert.NotNull(result); + var providers = result.ToList(); + Assert.Single(providers); // Should return exactly one consolidated provider + Assert.All(providers, provider => Assert.IsType(provider)); + + // Verify the provider has the expected name + var metadata = providers[0].CreateMetadata(); + Assert.Equal("get_azure_best_practices", metadata.Name); + Assert.Equal("get_azure_best_practices", metadata.Id); + } + + [Fact] + public async Task DiscoverServersAsync_WithReadOnlyFalse_CreatesNonReadOnlyProvider() + { + // Arrange + var options = new ServiceStartOptions { ReadOnly = false }; + var strategy = CreateStrategy(options: options); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + Assert.False(((CommandGroupServerProvider)providers[0]).ReadOnly); + } + + [Fact] + public async Task DiscoverServersAsync_WithReadOnlyTrue_CreatesReadOnlyProvider() + { + // Arrange + var options = new ServiceStartOptions { ReadOnly = true }; + var strategy = CreateStrategy(options: options); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + Assert.True(((CommandGroupServerProvider)providers[0]).ReadOnly); + } + + [Fact] + public async Task DiscoverServersAsync_WithNullReadOnlyOption_DefaultsToFalse() + { + // Arrange + var options = new ServiceStartOptions { ReadOnly = null }; + var strategy = CreateStrategy(options: options); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + Assert.False(((CommandGroupServerProvider)providers[0]).ReadOnly); + } + + [Fact] + public async Task DiscoverServersAsync_WithCustomEntryPoint_SetsEntryPointOnProvider() + { + // Arrange + var customEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); + var strategy = CreateStrategy(entryPoint: customEntryPoint); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + Assert.Equal(customEntryPoint, ((CommandGroupServerProvider)providers[0]).EntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_WithNullEntryPoint_UsesCurrentProcessExecutable() + { + // Arrange + var strategy = CreateStrategy(entryPoint: null); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + // When EntryPoint is set to null, CommandGroupServerProvider defaults to current process executable + var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; + var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; + Assert.NotNull(actualEntryPoint); + // Should be the current test process executable + Assert.Equal(currentProcessPath, actualEntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_WithEmptyEntryPoint_ProviderDefaultsToCurrentProcess() + { + // Arrange + var strategy = CreateStrategy(entryPoint: ""); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; + var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; + // CommandGroupServerProvider defaults empty/null to current process + Assert.Equal(currentProcessPath, actualEntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_WithWhitespaceEntryPoint_ProviderDefaultsToCurrentProcess() + { + // Arrange + var strategy = CreateStrategy(entryPoint: " "); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; + var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; + // CommandGroupServerProvider defaults whitespace to current process + Assert.Equal(currentProcessPath, actualEntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_ProviderHasCorrectMetadata() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + + var metadata = providers[0].CreateMetadata(); + Assert.NotNull(metadata); + Assert.Equal("get_azure_best_practices", metadata.Name); + Assert.Equal("get_azure_best_practices", metadata.Id); + Assert.Equal(metadata.Name, metadata.Id); // Should be the same + Assert.NotNull(metadata.Description); + Assert.Contains("Azure best practices", metadata.Description); + Assert.Contains("infrastructure schema", metadata.Description); + } + + [Fact] + public async Task DiscoverServersAsync_ProviderIsCommandGroupServerProviderType() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + Assert.IsType(providers[0]); + } + + [Fact] + public async Task DiscoverServersAsync_CanBeCalledMultipleTimes() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result1 = await strategy.DiscoverServersAsync(); + var result2 = await strategy.DiscoverServersAsync(); + + // Assert + Assert.NotNull(result1); + Assert.NotNull(result2); + + var providers1 = result1.ToList(); + var providers2 = result2.ToList(); + Assert.Equal(providers1.Count, providers2.Count); + Assert.Single(providers1); + Assert.Single(providers2); + + // Should return equivalent results + var metadata1 = providers1[0].CreateMetadata(); + var metadata2 = providers2[0].CreateMetadata(); + Assert.Equal(metadata1.Name, metadata2.Name); + Assert.Equal(metadata1.Id, metadata2.Id); + Assert.Equal(metadata1.Description, metadata2.Description); + } + + [Fact] + public async Task DiscoverServersAsync_RespectsServiceStartOptionsValues() + { + // Arrange + var options = new ServiceStartOptions + { + ReadOnly = true, + }; + var azmcpEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); + var strategy = CreateStrategy(options: options, entryPoint: azmcpEntryPoint); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + + var serverProvider = (CommandGroupServerProvider)providers[0]; + Assert.True(serverProvider.ReadOnly); + Assert.Equal(azmcpEntryPoint, serverProvider.EntryPoint); + } + + [Fact] + public async Task DiscoverServersAsync_ResultCountIsConsistent() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result1 = await strategy.DiscoverServersAsync(); + var result2 = await strategy.DiscoverServersAsync(); + + // Assert + var count1 = result1.Count(); + var count2 = result2.Count(); + Assert.Equal(count1, count2); + Assert.Equal(1, count1); // Should always have exactly one consolidated provider + } + + [Fact] + public async Task DiscoverServersAsync_InheritsFromBaseDiscoveryStrategy() + { + // Arrange + var strategy = CreateStrategy(); + + // Act & Assert + Assert.IsAssignableFrom(strategy); + Assert.IsAssignableFrom(strategy); + + // Should be able to call the interface method + var result = await strategy.DiscoverServersAsync(); + Assert.NotNull(result); + } + + [Fact] + public async Task DiscoverServersAsync_FiltersCommandsByCompositeToolMapped() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + + var provider = (CommandGroupServerProvider)providers[0]; + var metadata = provider.CreateMetadata(); + Assert.Equal("get_azure_best_practices", metadata.Name); + + // The strategy should only create a provider for commands with CompositeToolMapped = "get_azure_best_practices" + // The actual filtering logic is tested implicitly by verifying we get the expected provider name + } + + [Fact] + public async Task DiscoverServersAsync_CreatesCommandGroupWithCorrectToolMetadata() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + + // We can't directly access the CommandGroup's ToolMetadata from the provider, + // but we can verify the provider was created successfully and has the expected characteristics + var provider = (CommandGroupServerProvider)providers[0]; + Assert.NotNull(provider); + + var metadata = provider.CreateMetadata(); + Assert.Equal("get_azure_best_practices", metadata.Name); + } + + // Keep original tests for backward compatibility + [Fact] + public async Task ShouldDiscoverConsolidatedToolServer() + { + var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); + var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); + var logger = NSubstitute.Substitute.For>(); + var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger); + var result = await strategy.DiscoverServersAsync(); + Assert.NotNull(result); + var providers = result.ToList(); + Assert.Single(providers); + } + + [Fact] + public async Task ShouldDiscoverConsolidatedToolServer_SetsPropertiesCorrectly() + { + var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); + var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions { ReadOnly = true }); + var azmcpEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); + var logger = NSubstitute.Substitute.For>(); + var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger) + { + EntryPoint = azmcpEntryPoint + }; + var result = (await strategy.DiscoverServersAsync()).ToList(); + Assert.Single(result); + + // Should have the consolidated tool provider with expected name + var provider = result[0]; + Assert.Equal("get_azure_best_practices", provider.CreateMetadata().Name); + + // Should set ReadOnly and EntryPoint as expected + var serverProvider = (CommandGroupServerProvider)provider; + Assert.True(serverProvider.ReadOnly); + Assert.Equal(azmcpEntryPoint, serverProvider.EntryPoint); + } + + [Fact] + public void GetAzmcpExecutablePath_ReturnsCorrectPathForCurrentOS() + { + // Arrange & Act + var azmcpPath = McpTestUtilities.GetAzMcpExecutablePath(); + + // Assert + Assert.NotNull(azmcpPath); + Assert.NotEmpty(azmcpPath); + + // Should end with the correct executable name for the current OS + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform( + System.Runtime.InteropServices.OSPlatform.Windows)) + { + Assert.EndsWith("azmcp.exe", azmcpPath); + } + else + { + Assert.EndsWith("azmcp", azmcpPath); + Assert.False(azmcpPath.EndsWith("azmcp.exe")); + } + + // Should be in the same directory as the test assembly + var testAssemblyPath = System.Reflection.Assembly.GetExecutingAssembly().Location; + var testDirectory = Path.GetDirectoryName(testAssemblyPath); + var expectedDirectory = Path.GetDirectoryName(azmcpPath); + Assert.Equal(testDirectory, expectedDirectory); + } + + [Fact] + public async Task DiscoverServersAsync_AlwaysReturnsOneProvider() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + // Unlike CommandGroupDiscoveryStrategy which returns multiple providers (one per command group), + // ConsolidatedToolDiscoveryStrategy always returns exactly one provider that consolidates + // all commands with the same CompositeToolMapped value + Assert.Single(providers); + } + + [Fact] + public async Task DiscoverServersAsync_ProviderNameMatchesCompositeToolMappedValue() + { + // Arrange + var strategy = CreateStrategy(); + + // Act + var result = await strategy.DiscoverServersAsync(); + + // Assert + var providers = result.ToList(); + Assert.Single(providers); + + var metadata = providers[0].CreateMetadata(); + // The provider name should match the CompositeToolMapped value used in the filtering logic + Assert.Equal("get_azure_best_practices", metadata.Name); + } +} \ No newline at end of file diff --git a/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs b/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs index 0e8d514670..86527a39cc 100644 --- a/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs +++ b/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs @@ -11,7 +11,7 @@ namespace Azure.Mcp.Tools.AzureBestPractices; public class AzureBestPracticesSetup : IAreaSetup { - public string Name => "get_bestpractices"; + public string Name => "get-bestpractices"; public void ConfigureServices(IServiceCollection services) { diff --git a/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs b/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs index 98c2307f0f..366ff10c9d 100644 --- a/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs +++ b/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs @@ -44,6 +44,8 @@ public sealed class BestPracticesCommand(ILogger logger) : Secret = false }; + public override string CompositeToolMapped => "get_azure_best_practices"; + protected override void RegisterOptions(Command command) { command.Options.Add(_resourceOption); diff --git a/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs b/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs index 99c217c34c..031693a814 100644 --- a/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs +++ b/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs @@ -42,6 +42,8 @@ private static string LoadBestPracticesText() Secret = false }; + public override string CompositeToolMapped => "get_azure_best_practices"; + public override Task ExecuteAsync(CommandContext context, ParseResult parseResult) { var bestPractices = GetBestPracticesText(); diff --git a/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs b/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs index c02e553884..3bf4719ef0 100644 --- a/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs +++ b/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs @@ -41,6 +41,8 @@ Always use the Bicep schema to verify the available property names and values wh Secret = false }; + public override string CompositeToolMapped => "get_azure_best_practices"; + private static readonly Lazy s_serviceProvider; static BicepSchemaGetCommand() From 06c03bb5334965a3b02b3d2219353cdf3f65c322 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Mon, 13 Oct 2025 13:12:18 -0400 Subject: [PATCH 02/16] Make consolidated mode work end-to-end with all the tools --- .../Discovery/BaseDiscoveryStrategy.cs | 10 +- .../Discovery/CommandGroupServerProvider.cs | 3 +- .../ConsolidatedToolDiscoveryStrategy.cs | 172 +++- .../ConsolidatedToolServerProvider.cs | 96 ++ .../Commands/ToolLoading/ServerToolLoader.cs | 11 +- .../Models/ConsolidatedToolDefinition.cs | 6 + .../Server/Resources/consolidated-tools.json | 948 ++++++++++++++++++ .../src/Areas/Server/ServerJsonContext.cs | 2 + .../src/Commands/BaseCommand.cs | 1 - .../src/Commands/IBaseCommand.cs | 7 - .../Areas/Server/ServerCommandTests.cs | 184 +++- .../ConsolidatedToolDiscoveryStrategyTests.cs | 491 +-------- .../src/AzureBestPracticesSetup.cs | 2 +- .../src/Commands/BestPracticesCommand.cs | 2 - .../AzureTerraformBestPracticesGetCommand.cs | 1 - .../src/Commands/BicepSchemaGetCommand.cs | 2 - .../Commands/Admin/AdminSettingsGetCommand.cs | 2 +- .../FileSystem/FileSystemCreateCommand.cs | 2 +- .../src/Commands/Server/ServerListCommand.cs | 2 +- 19 files changed, 1382 insertions(+), 562 deletions(-) create mode 100644 core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolServerProvider.cs create mode 100644 core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs index 92fed40b91..3fb0276113 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/BaseDiscoveryStrategy.cs @@ -72,13 +72,13 @@ public async Task GetOrCreateClientAsync(string name, McpClientOption throw new ArgumentNullException(nameof(name), "Server name cannot be null or empty."); } - // if (_clientCache.TryGetValue(name, out var client)) - // { - // return client; - // } + if (_clientCache.TryGetValue(name, out var client)) + { + return client; + } var serverProvider = await FindServerProviderAsync(name); - var client = await serverProvider.CreateClientAsync(clientOptions ?? new McpClientOptions()); + client = await serverProvider.CreateClientAsync(clientOptions ?? new McpClientOptions()); _clientCache[name] = client; return client; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs index 638341233e..22e42143b1 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/CommandGroupServerProvider.cs @@ -80,8 +80,7 @@ public McpServerMetadata CreateMetadata() { Id = _commandGroup.Name, Name = _commandGroup.Name, - Description = _commandGroup.Description, - ToolMetadata = _commandGroup.ToolMetadata, + Description = _commandGroup.Description }; } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 98d98e6efa..0a8461de3a 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Reflection; +using System.Text.Json; +using Azure.Mcp.Core.Areas.Server.Models; using Azure.Mcp.Core.Areas.Server.Options; using Azure.Mcp.Core.Commands; using Microsoft.Extensions.Logging; @@ -25,6 +28,7 @@ public sealed class ConsolidatedToolDiscoveryStrategy(CommandFactory commandFact /// This can be used to specify a custom entry point for the commands. /// public string? EntryPoint { get; set; } = null; + public static readonly string[] IgnoredCommandGroups = ["server", "tools"]; /// /// Discovers available command groups and converts them to MCP server providers. @@ -32,52 +36,140 @@ public sealed class ConsolidatedToolDiscoveryStrategy(CommandFactory commandFact /// A collection of command group server providers. public override Task> DiscoverServersAsync() { - // Find all commands with the same CompositeToolMapped value + // Load consolidated tools from JSON file + var consolidatedTools = new List(); + try + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = "Azure.Mcp.Core.Areas.Server.Resources.consolidated-tools.json"; + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream != null) + { + using var reader = new StreamReader(stream); + var json = reader.ReadToEnd(); + var jsonDoc = JsonDocument.Parse(json); + if (jsonDoc.RootElement.TryGetProperty("consolidated_tools", out var toolsArray)) + { + consolidatedTools = JsonSerializer.Deserialize(toolsArray.GetRawText(), ServerJsonContext.Default.ListConsolidatedToolDefinition) ?? new List(); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load consolidated tools from JSON file"); + return Task.FromResult>(new List()); + } + + var providers = new List(); var allCommands = _commandFactory.AllCommands; - var matchingCommands = allCommands - .Where(kvp => !string.IsNullOrWhiteSpace(kvp.Value.CompositeToolMapped) && - string.Equals(kvp.Value.CompositeToolMapped, "get_azure_best_practices", StringComparison.OrdinalIgnoreCase)) - .ToList(); - - // Create a new CommandGroup and add the matching commands - var commandGroup = new CommandGroup("get_azure_best_practices", "Retrieve Azure best practices and infrastructure schema for code generation, deployment, and operations. Covers general Azure practices, Azure Functions best practices, Terraform configurations, Bicep template schemas, and deployment best practices."); - _commandFactory.RootGroup.AddSubGroup(commandGroup); - foreach (var (commandName, command) in matchingCommands) + + // Filter out commands that belong to ignored command groups + var filteredCommands = allCommands + .Where(kvp => + { + var serviceArea = _commandFactory.GetServiceArea(kvp.Key); + return serviceArea == null || !IgnoredCommandGroups.Contains(serviceArea, StringComparer.OrdinalIgnoreCase); + }) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + + // Track unmatched commands + var unmatchedCommands = new HashSet(filteredCommands.Keys, StringComparer.OrdinalIgnoreCase); + + // Iterate through each consolidated tool definition + foreach (var consolidatedTool in consolidatedTools) { - commandGroup.AddCommand(commandName, command); - // Extract just the command name (remove any prefix) - // TODO: It is going to be all get commands??? - // var simpleName = commandName.Replace("azmcp_", "").Replace("_", "."); - // commandGroup.AddCommand(simpleName, command); + // Find all commands that match this consolidated tool's mapped tool list + var matchingCommands = filteredCommands + .Where(kvp => consolidatedTool.MappedToolList != null && + consolidatedTool.MappedToolList.Contains(kvp.Key, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + if (matchingCommands.Count == 0) + { + continue; + } + + // Create a new CommandGroup and add the matching commands + var commandGroup = new CommandGroup(consolidatedTool.Name, consolidatedTool.Description); + _commandFactory.RootGroup.AddSubGroup(commandGroup); + + foreach (var (commandName, command) in matchingCommands) + { + // Validate that the command's metadata matches the consolidated tool's metadata + if (!AreMetadataEqual(command.Metadata, consolidatedTool.ToolMetadata)) + { + var errorMessage = $"Command '{commandName}' has mismatched ToolMetadata for consolidated tool '{consolidatedTool.Name}'. " + + $"Command metadata: [Destructive={command.Metadata.Destructive}, Idempotent={command.Metadata.Idempotent}, " + + $"OpenWorld={command.Metadata.OpenWorld}, ReadOnly={command.Metadata.ReadOnly}, Secret={command.Metadata.Secret}, " + + $"LocalRequired={command.Metadata.LocalRequired}], " + + $"Consolidated tool metadata: [Destructive={consolidatedTool.ToolMetadata?.Destructive}, " + + $"Idempotent={consolidatedTool.ToolMetadata?.Idempotent}, OpenWorld={consolidatedTool.ToolMetadata?.OpenWorld}, " + + $"ReadOnly={consolidatedTool.ToolMetadata?.ReadOnly}, Secret={consolidatedTool.ToolMetadata?.Secret}, " + + $"LocalRequired={consolidatedTool.ToolMetadata?.LocalRequired}]"; + _logger.LogError(errorMessage); + // throw new InvalidOperationException(errorMessage); + } + + commandGroup.AddCommand(commandName, command); + // Remove matched commands from the unmatched list + unmatchedCommands.Remove(commandName); + } + + commandGroup.ToolMetadata = consolidatedTool.ToolMetadata; + + ConsolidatedToolServerProvider serverProvider = new ConsolidatedToolServerProvider(commandGroup) + { + ReadOnly = _options.Value.ReadOnly ?? false, + EntryPoint = EntryPoint, + }; + + providers.Add(serverProvider); } - commandGroup.ToolMetadata = new ToolMetadata + +#if DEBUG + // In debug mode, throw an error if there are unmatched commands + if (unmatchedCommands.Count > 0) { - Destructive = false, - Idempotent = true, - OpenWorld = false, - ReadOnly = true, - LocalRequired = false, - Secret = false - }; - - CommandGroupServerProvider serverProvider = new CommandGroupServerProvider(commandGroup) + var unmatchedList = string.Join(", ", unmatchedCommands.OrderBy(c => c)); + var errorMessage = $"Found {unmatchedCommands.Count} unmatched commands: {unmatchedList}"; + _logger.LogError(errorMessage); + throw new InvalidOperationException(errorMessage); + } +#else + // In release mode, just log a warning + if (unmatchedCommands.Count > 0) { - ReadOnly = _options.Value.ReadOnly ?? false, - EntryPoint = EntryPoint, - }; - - var providers = new List { serverProvider }; - // providers.AddRange(_commandFactory.RootGroup.SubGroup - // .Where(group => _options.Value.Namespace == null || - // _options.Value.Namespace.Length == 0 || - // _options.Value.Namespace.Contains(group.Name, StringComparer.OrdinalIgnoreCase)) - // .Select(group => new CommandGroupServerProvider(group) - // { - // ReadOnly = _options.Value.ReadOnly ?? false, - // EntryPoint = EntryPoint, - // }) - // .Cast(); + var unmatchedList = string.Join(", ", unmatchedCommands.OrderBy(c => c)); + _logger.LogWarning("Found {Count} unmatched commands: {Commands}", unmatchedCommands.Count, unmatchedList); + } +#endif return Task.FromResult>(providers); } + + /// + /// Compares two ToolMetadata objects for equality. + /// + /// The first ToolMetadata to compare. + /// The second ToolMetadata to compare. + /// True if the metadata objects are equal, false otherwise. + private static bool AreMetadataEqual(ToolMetadata? metadata1, ToolMetadata? metadata2) + { + if (metadata1 == null && metadata2 == null) + { + return true; + } + + if (metadata1 == null || metadata2 == null) + { + return false; + } + + return metadata1.Destructive == metadata2.Destructive && + metadata1.Idempotent == metadata2.Idempotent && + metadata1.OpenWorld == metadata2.OpenWorld && + metadata1.ReadOnly == metadata2.ReadOnly && + metadata1.Secret == metadata2.Secret && + metadata1.LocalRequired == metadata2.LocalRequired; + } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolServerProvider.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolServerProvider.cs new file mode 100644 index 0000000000..f0d94bd066 --- /dev/null +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolServerProvider.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Mcp.Core.Areas.Server.Options; +using Azure.Mcp.Core.Commands; +using ModelContextProtocol.Client; + +namespace Azure.Mcp.Core.Areas.Server.Commands.Discovery; + +/// +/// Server provider that starts the azmcp server in "all" mode while explicitly +/// enumerating each tool (command) in a command group using repeated --tool flags. +/// This allows selective exposure of only the commands that belong to the provided group +/// without relying on the namespace grouping mechanism. +/// +public sealed class ConsolidatedToolServerProvider(CommandGroup commandGroup) : IMcpServerProvider +{ + private readonly CommandGroup _commandGroup = commandGroup; + private string? _entryPoint = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; + + /// + /// Gets or sets the entry point executable path for the MCP server. + /// If set to null or empty, defaults to the current process executable. + /// + public string? EntryPoint + { + get => _entryPoint; + set => _entryPoint = string.IsNullOrWhiteSpace(value) + ? System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName + : value; + } + + /// + /// Gets or sets whether the MCP server should run in read-only mode. + /// + public bool ReadOnly { get; set; } = false; + + /// + /// Creates an MCP client from a command group. + /// + public async Task CreateClientAsync(McpClientOptions clientOptions) + { + if (string.IsNullOrWhiteSpace(EntryPoint)) + { + throw new InvalidOperationException("EntryPoint must be set before creating the MCP client."); + } + + var arguments = BuildArguments(); + + var transportOptions = new StdioClientTransportOptions + { + Name = _commandGroup.Name, + Command = EntryPoint, + Arguments = arguments, + }; + + var clientTransport = new StdioClientTransport(transportOptions); + return await McpClient.CreateAsync(clientTransport, clientOptions); + } + + /// + /// Builds the command-line arguments for the MCP server process. + /// Pattern: server start --mode all (--tool )+ [--read-only] + /// + internal string[] BuildArguments() + { + var arguments = new List { "server", "start", "--mode", "all" }; + + foreach (var kvp in _commandGroup.Commands) + { + arguments.Add("--tool"); + arguments.Add(kvp.Key); + } + + if (ReadOnly) + { + arguments.Add($"--{ServiceOptionDefinitions.ReadOnlyName}"); + } + + return [.. arguments]; + } + + /// + /// Creates metadata for the MCP server provider based on the command group. + /// + public McpServerMetadata CreateMetadata() + { + return new McpServerMetadata + { + Id = _commandGroup.Name, + Name = _commandGroup.Name, + Description = _commandGroup.Description, + ToolMetadata = _commandGroup.ToolMetadata, + }; + } +} diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs index cdd6a63ea7..d7c656cafb 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/ToolLoading/ServerToolLoader.cs @@ -90,7 +90,6 @@ Sub commands are routed to MCP servers that require specific fields inside the " IdempotentHint = metadata.ToolMetadata.Idempotent, OpenWorldHint = metadata.ToolMetadata.OpenWorld, ReadOnlyHint = metadata.ToolMetadata.ReadOnly, - // Secret and LocalRequired are not annotated as hints }; } @@ -354,10 +353,10 @@ private async Task InvokeToolLearn(RequestContext private async Task> GetChildToolListAsync(RequestContext request, string tool) { - // if (_cachedToolLists.TryGetValue(tool, out var cachedList)) - // { - // return cachedList; - // } + if (_cachedToolLists.TryGetValue(tool, out var cachedList)) + { + return cachedList; + } if (string.IsNullOrWhiteSpace(request.Params?.Name)) { @@ -381,8 +380,6 @@ private async Task> GetChildToolListAsync(RequestContext t.ProtocolTool) .Where(t => !(options?.Value?.ReadOnly ?? false) || (t.Annotations?.ReadOnlyHint == true)) - // SecretHint doesn't exist yet - // .Where(t => !(options?.Value?.Secret ?? false) || (t.Annotations?.SecretHint == true)) .ToList(); _cachedToolLists[tool] = list; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs index 57a3264d84..3bd6cdcad4 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs @@ -29,4 +29,10 @@ public sealed class ConsolidatedToolDefinition /// [JsonPropertyName("toolMetadata")] public ToolMetadata? ToolMetadata { get; set; } + + /// + /// Gets or sets the list of tool names that are mapped to this consolidated tool. + /// + [JsonPropertyName("mappedToolList")] + public List MappedToolList { get; set; } = new List(); } \ No newline at end of file diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json new file mode 100644 index 0000000000..35a83587aa --- /dev/null +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -0,0 +1,948 @@ +{ + "consolidated_tools": [ + { + "name": "get_azure_subscriptions_and_resource_groups", + "description": "Get information about Azure subscriptions and resource groups that the user has access to.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_group_list", + "azmcp_subscription_list" + ] + }, + { + "name": "get_azure_app_resource_details", + "description": "Get details about Azure application platform services, such as Azure Functions.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_functionapp_get" + ] + }, + { + "name": "add_azure_app_service_database", + "description": "Add and configure database integrations for Azure App Service applications.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": true, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_appservice_database_add" + ] + }, + { + "name": "get_azure_databases_details", + "description": "Comprehensive Azure database management tool for MySQL, PostgreSQL, SQL Database, SQL Server, and Cosmos DB. List and query databases, retrieve server configurations and parameters, explore table schemas, execute database queries, manage Cosmos DB containers and items, list and view detailed information about SQL servers, and view database server details across all Azure database services.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_mysql_database_list", + "azmcp_mysql_database_query", + "azmcp_mysql_server_config_get", + "azmcp_mysql_server_list", + "azmcp_mysql_server_param_get", + "azmcp_mysql_table_list", + "azmcp_mysql_table_schema_get", + "azmcp_postgres_database_list", + "azmcp_postgres_database_query", + "azmcp_postgres_server_config_get", + "azmcp_postgres_server_list", + "azmcp_postgres_server_param_get", + "azmcp_postgres_table_list", + "azmcp_postgres_table_schema_get", + "azmcp_sql_db_list", + "azmcp_sql_db_show", + "azmcp_sql_server_list", + "azmcp_sql_server_show", + "azmcp_cosmos_account_list", + "azmcp_cosmos_database_container_item_query", + "azmcp_cosmos_database_container_list", + "azmcp_cosmos_database_list" + ] + }, + { + "name": "create_azure_sql_databases_and_servers", + "description": "Create new Azure SQL databases and SQL servers with configurable performance tiers and settings.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_db_create", + "azmcp_sql_server_create" + ] + }, + { + "name": "rename_azure_sql_databases", + "description": "Rename Azure SQL databases to a new name within the same SQL server.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_db_rename" + ] + }, + { + "name": "edit_azure_sql_databases_and_servers", + "description": "Update and delete Azure SQL databases and SQL servers. Modify database configurations or permanently remove servers and databases.", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_db_update", + "azmcp_sql_db_delete", + "azmcp_sql_server_delete" + ] + }, + { + "name": "edit_azure_databases", + "description": "Edit Azure MySQL and PostgreSQL database server parameters", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_mysql_server_param_set", + "azmcp_postgres_server_param_set" + ] + }, + { + "name": "get_azure_resource_and_app_health_status", + "description": "Get Azure resource and application health status, metrics, availability, and service health events. Query Log Analytics, list Log Analytics workspaces, get the current availability status of Azure resources, list service health events and incidents, list Grafana instances, view Datadog monitored resources, perform App Lens diagnostics for comprehensive application troubleshooting, and retrieve Application Insights recommendations for performance optimization.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_applicationinsights_recommendation_list", + "azmcp_applens_resource_diagnose", + "azmcp_grafana_list", + "azmcp_datadog_monitoredresources_list", + "azmcp_monitor_workspace_list", + "azmcp_monitor_healthmodels_entity_gethealth", + "azmcp_monitor_metrics_definitions", + "azmcp_monitor_metrics_query", + "azmcp_monitor_resource_log_query", + "azmcp_monitor_table_list", + "azmcp_monitor_table_type_list", + "azmcp_monitor_workspace_log_query", + "azmcp_resourcehealth_availability-status_get", + "azmcp_resourcehealth_availability-status_list", + "azmcp_resourcehealth_service-health-events_list" + ] + }, + { + "name": "deploy_resources_and_applications_to_azure", + "description": "Deploy resources and applications to Azure. Retrieve application logs, access Bicep and Terraform rules, get CI/CD pipeline guidance, and generate deployment plans.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_deploy_app_logs_get", + "azmcp_deploy_iac_rules_get", + "azmcp_deploy_pipeline_guidance_get", + "azmcp_deploy_plan_get" + ] + }, + { + "name": "execute_azure_developer_cli", + "description": "Execute Azure Developer CLI (azd) commands for modern cloud-native application development workflows. Supports azd operations including project initialization, environment provisioning, application deployment, monitoring setup, and cleanup. Essential for cloud-native development teams using azd templates and Infrastructure as Code patterns with integrated CI/CD pipeline automation.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_extension_azd" + ] + }, + { + "name": "get_azure_app_config_settings", + "description": "Get details about Azure App Configuration settings", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_appconfig_account_list", + "azmcp_appconfig_kv_get" + ] + }, + { + "name": "edit_azure_app_config_settings", + "description": "Delete or set Azure App Configuration settings with write operations.", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_appconfig_kv_delete", + "azmcp_appconfig_kv_set" + ] + }, + { + "name": "lock_unlock_azure_app_config_settings", + "description": "Lock and unlock Azure App Configuration settings", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_appconfig_kv_lock_set" + ] + }, + { + "name": "edit_azure_workbooks", + "description": "Update or delete Azure Monitor Workbooks", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_workbooks_delete", + "azmcp_workbooks_update" + ] + }, + { + "name": "create_azure_workbooks", + "description": "Create new Azure Monitor Workbooks", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_workbooks_create" + ] + }, + { + "name": "get_azure_workbooks_details", + "description": "Get details about Azure Monitor Workbooks including listing workbooks and viewing specific workbook configurations.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_workbooks_list", + "azmcp_workbooks_show" + ] + }, + { + "name": "audit_azure_resources_compliance", + "description": "Generate compliance and security audit reports for Azure resources using Azure Quick Review (azqr).", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_extension_azqr" + ] + }, + { + "name": "execute_azure_cli", + "description": "Answer questions about an Azure environment by executing Azure CLI commands", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_extension_az" + ] + }, + { + "name": "get_azure_security_configurations", + "description": "List Azure RBAC role assignments", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_role_assignment_list" + ] + }, + { + "name": "get_azure_key_vault_items", + "description": "View and retrieve Azure Key Vault security artifacts, including certificates, keys (both listing and individual key details), secret names and properties (listing only, not secret values), and admin settings.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_keyvault_certificate_get", + "azmcp_keyvault_certificate_list", + "azmcp_keyvault_key_get", + "azmcp_keyvault_key_list", + "azmcp_keyvault_secret_list", + "azmcp_keyvault_admin_settings_get" + ] + }, + { + "name": "get_azure_key_vault_secret_values", + "description": "Retrieve the actual secret values from Azure Key Vault. Use this tool when you need to access the sensitive content of a secret, not just its metadata or properties.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": true, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_keyvault_secret_get" + ] + }, + { + "name": "create_azure_key_vault_items", + "description": "Create new security artifacts in Azure Key Vault including certificates and keys.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_keyvault_certificate_create", + "azmcp_keyvault_key_create" + ] + }, + { + "name": "create_azure_key_vault_secrets", + "description": "Create new secrets in Azure Key Vault.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": true, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_keyvault_secret_create" + ] + }, + { + "name": "import_azure_key_vault_certificates", + "description": "Import external certificates into Azure Key Vault", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": true + }, + "mappedToolList": [ + "azmcp_keyvault_certificate_import" + ] + }, + { + "name": "get_azure_best_practices", + "description": "Retrieve Azure best practices and infrastructure schema for code generation, deployment, and operations. Covers general Azure practices, Azure Functions best practices, Terraform configurations, Bicep template schemas, and deployment best practices.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_azureterraformbestpractices_get", + "azmcp_bicepschema_get", + "azmcp_get_bestpractices_get" + ] + }, + { + "name": "design_azure_architecture", + "description": "Comprehensive Azure architecture design and visualization tool. Provide cloud architecture consulting and recommend optimal Azure solutions following Well-Architected Framework principles. Also generates visual Mermaid diagrams from application topologies.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_cloudarchitect_design", + "azmcp_deploy_architecture_diagram_generate" + ] + }, + { + "name": "get_azure_load_testing_details", + "description": "Get Azure Load Testing test configurations, results, and test resources including listing load testing resources.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_loadtesting_testresource_list", + "azmcp_loadtesting_test_get", + "azmcp_loadtesting_testrun_get", + "azmcp_loadtesting_testrun_list" + ] + }, + { + "name": "create_azure_load_testing", + "description": "Create Load Testing resource or execute Azure Load Testing tests.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_loadtesting_test_create", + "azmcp_loadtesting_testresource_create", + "azmcp_loadtesting_testrun_create" + ] + }, + { + "name": "update_azure_load_testing_configurations", + "description": "Update Azure Load Testing configurations and test run settings.", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_loadtesting_testrun_update" + ] + }, + { + "name": "get_azure_ai_resources_details", + "description": "Get details about Azure AI resources including listing and querying AI Search services, listing models available to be deployed and models deployed already, knowledge index schema by AI Foundry, and listing AI Foundry agents.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_search_service_list", + "azmcp_search_index_get", + "azmcp_search_index_query", + "azmcp_foundry_models_deployments_list", + "azmcp_foundry_models_list", + "azmcp_foundry_knowledge_index_list", + "azmcp_foundry_knowledge_index_schema", + "azmcp_foundry_openai_models-list", + "azmcp_foundry_agents_list" + ] + }, + { + "name": "use_azure_openai_models", + "description": "Generate text completions, chat responses, and embeddings using Azure OpenAI models in AI Foundry. Create conversational AI interactions and vector embeddings for semantic search and analysis.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_foundry_openai_create-completion", + "azmcp_foundry_openai_embeddings-create", + "azmcp_foundry_openai_chat-completions-create" + ] + }, + { + "name": "deploy_azure_ai_models", + "description": "Deploy a model to Azure AI Foundry.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_foundry_models_deploy" + ] + }, + { + "name": "connect_azure_ai_foundry_agents", + "description": "Connect to Azure AI Foundry agents for establishing agent connections and communication channels.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": true, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_foundry_agents_connect" + ] + }, + { + "name": "query_and_evaluate_azure_ai_foundry_agents", + "description": "Query Azure AI Foundry agents with prompts and evaluate their responses using various metrics for comprehensive performance assessment.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": true, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_foundry_agents_query-and-evaluate" + ] + }, + { + "name": "evaluate_azure_ai_foundry_agents", + "description": "Evaluate Azure AI Foundry agents for performance assessment and testing of agent capabilities using evaluation metrics.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_foundry_agents_evaluate" + ] + }, + { + "name": "get_azure_storage_details", + "description": "Get details about Azure Storage resources including Storage accounts, blob containers, blob data, and Azure Managed Lustre filesystem information including available SKUs, subnet size validation, and subnet size requirements.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_managedlustre_filesystem_list", + "azmcp_managedlustre_filesystem_sku_get", + "azmcp_storage_account_get", + "azmcp_storage_blob_container_get", + "azmcp_storage_blob_get", + "azmcp_managedlustre_filesystem_subnetsize_ask", + "azmcp_managedlustre_filesystem_subnetsize_validate" + ] + }, + { + "name": "create_azure_storage", + "description": "Create Azure Storage accounts, blob containers, and Azure Managed Lustre filesystems.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_storage_account_create", + "azmcp_storage_blob_container_create", + "azmcp_managedlustre_filesystem_create" + ] + }, + { + "name": "update_azure_managed_lustre_filesystems", + "description": "Update existing Azure Managed Lustre filesystem configurations.", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_managedlustre_filesystem_update" + ] + }, + { + "name": "upload_azure_storage_blobs", + "description": "Upload files and data to Azure Storage blob containers.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": true + }, + "mappedToolList": [ + "azmcp_storage_blob_upload" + ] + }, + { + "name": "get_azure_cache_for_redis_details", + "description": "Get details about Azure Cache for Redis resources including cache instances, clusters, access policies, and database configurations. List and manage Redis caches and clusters.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_redis_cache_list", + "azmcp_redis_cluster_list", + "azmcp_redis_cache_accesspolicy_list", + "azmcp_redis_cluster_database_list" + ] + }, + { + "name": "browse_azure_marketplace_products", + "description": "Browse products and offers in the Azure Marketplace", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_marketplace_product_list", + "azmcp_marketplace_product_get" + ] + }, + { + "name": "get_azure_capacity", + "description": "Check Azure resource capacity, quotas, available regions, usage limits, and high-performance computing infrastructure", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_quota_region_availability_list", + "azmcp_quota_usage_check", + "azmcp_managedlustre_filesystem_required-subnet-size" + ] + }, + { + "name": "get_azure_messaging_service_details", + "description": "Get details about Azure messaging services including Service Bus queues, topics, subscriptions, and Event Grid topics and subscriptions.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_eventgrid_topic_list", + "azmcp_eventgrid_subscription_list", + "azmcp_servicebus_queue_details", + "azmcp_servicebus_topic_details", + "azmcp_servicebus_topic_subscription_details" + ] + }, + { + "name": "get_azure_data_analytics_details", + "description": "Get details about Azure Event Hubs namespaces and streaming event ingestion resources.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_eventhubs_namespace_get" + ] + }, + { + "name": "publish_azure_eventgrid_events", + "description": "Publish custom events to Azure Event Grid topics for event-driven architectures with schema validation and delivery guarantees.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_eventgrid_events_publish" + ] + }, + { + "name": "get_azure_data_explorer_kusto_details", + "description": "Get details about Azure Data Explorer (Kusto). List clusters, execute KQL queries, manage databases, explore table schemas, and get data samples.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_kusto_cluster_list", + "azmcp_kusto_cluster_get", + "azmcp_kusto_database_list", + "azmcp_kusto_query", + "azmcp_kusto_sample", + "azmcp_kusto_table_list", + "azmcp_kusto_table_schema" + ] + }, + { + "name": "create_azure_database_admin_configurations", + "description": "Create Azure SQL Server firewall rules", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_server_firewall-rule_create" + ] + }, + { + "name": "delete_azure_database_admin_configurations", + "description": "Delete Azure SQL Server firewall rules", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_server_firewall-rule_delete" + ] + }, + { + "name": "get_azure_database_admin_configuration_details", + "description": "Get details about Azure SQL Server Administration configurations including elastic pools, Entra admin assignments, and firewall rules.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_sql_elastic-pool_list", + "azmcp_sql_server_entra-admin_list", + "azmcp_sql_server_firewall-rule_list" + ] + }, + { + "name": "get_azure_container_details", + "description": "Get details about Azure container services including Azure Container Registry (ACR) and Azure Kubernetes Service (AKS). View registries, repositories, nodepools, clusters, cluster configurations, and individual nodepool details.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_acr_registry_list", + "azmcp_acr_registry_repository_list", + "azmcp_aks_cluster_list", + "azmcp_aks_cluster_get", + "azmcp_aks_nodepool_list", + "azmcp_aks_nodepool_get" + ] + }, + { + "name": "get_azure_virtual_desktop_details", + "description": "Get details about Azure Virtual Desktop resources including host pools, session hosts, and user sessions.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_virtualdesktop_hostpool_list", + "azmcp_virtualdesktop_hostpool_sessionhost_list", + "azmcp_virtualdesktop_hostpool_sessionhost_usersession-list" + ] + }, + { + "name": "get_azure_signalr_details", + "description": "Get details about Azure SignalR Service resources including runtime information, identity, network ACLs, and upstream templates.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_signalr_runtime_get" + ] + }, + { + "name": "append_azure_confidential_ledger_entries", + "description": "Append tamper-proof entries to Azure Confidential Ledger instances and retrieve transaction identifiers.", + "toolMetadata": { + "destructive": false, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_confidentialledger_entries_append" + ] + }, + { + "name": "send_azure_communication_sms", + "description": "Send SMS messages to one or more recipients using Azure Communication Services with delivery status tracking.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": true, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_communication_sms_send" + ] + }, + { + "name": "recognize_speech_from_audio", + "description": "Convert speech from audio files to text using Azure AI Services Speech recognition. Supports various audio formats including WAV, MP3, OPUS/OGG, FLAC, and more.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": true + }, + "mappedToolList": [ + "azmcp_speech_stt_recognize" + ] + } + ] +} \ No newline at end of file diff --git a/core/Azure.Mcp.Core/src/Areas/Server/ServerJsonContext.cs b/core/Azure.Mcp.Core/src/Areas/Server/ServerJsonContext.cs index 1c86395d69..6bf6b2e521 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/ServerJsonContext.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/ServerJsonContext.cs @@ -19,6 +19,8 @@ namespace Azure.Mcp.Core.Areas.Server; [JsonSerializable(typeof(List))] [JsonSerializable(typeof(ToolInputSchema))] [JsonSerializable(typeof(ToolPropertySchema))] +[JsonSerializable(typeof(ConsolidatedToolDefinition))] +[JsonSerializable(typeof(List))] [JsonSourceGenerationOptions( PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, diff --git a/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs b/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs index 9ac1f206c0..c74aa23c8d 100644 --- a/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs +++ b/core/Azure.Mcp.Core/src/Commands/BaseCommand.cs @@ -29,7 +29,6 @@ protected BaseCommand() public abstract string Description { get; } public abstract string Title { get; } public abstract ToolMetadata Metadata { get; } - public virtual string? CompositeToolMapped { get; protected set; } protected virtual void RegisterOptions(Command command) { diff --git a/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs b/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs index 1bac5a90c1..f61555dcdb 100644 --- a/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs +++ b/core/Azure.Mcp.Core/src/Commands/IBaseCommand.cs @@ -33,13 +33,6 @@ public interface IBaseCommand /// ToolMetadata Metadata { get; } - /// - /// Gets the name of the composite tool that this command maps to. - /// Used for discovery and routing in the consolidated tool system. - /// If null, the command is not mapped to any composite tool. - /// - string? CompositeToolMapped { get; } - /// /// Gets the command definition /// diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs index a4060d716b..fe432cf36f 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs @@ -193,7 +193,7 @@ public async Task AllMode_LoadsAllIndividualTools() public async Task SingleProxyMode_LoadsSingleAzureTool() { // Arrange - await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated"); + await using var client = await CreateClientAsync("server", "start", "--mode", "single"); // Act var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); @@ -577,6 +577,188 @@ public async Task VerifyUniqueToolNames_InDefaultMode() #endregion + #region Consolidated Proxy Mode Tests + + [Fact] + public async Task ConsolidatedProxyMode_LoadsConsolidatedTools() + { + // Arrange + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated"); + + // Act + var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.NotEmpty(listResult); + + var toolNames = listResult.Select(t => t.Name).ToList(); + + // In consolidated mode, should have consolidated tools grouping related operations + Assert.True(toolNames.Count > 20, $"Expected more than 20 consolidated tools, got {toolNames.Count}"); + + // Should include some known consolidated tools + Assert.Contains(toolNames, name => name.Contains("azure_subscriptions_and_resource_groups", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(toolNames, name => name.Contains("azure_databases_details", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(toolNames, name => name.Contains("azure_storage_details", StringComparison.OrdinalIgnoreCase)); + + Output.WriteLine($"Consolidated proxy mode loaded {toolNames.Count} tools"); + foreach (var name in toolNames) + { + Output.WriteLine($" - {name}"); + } + } + + [Fact] + public async Task ConsolidatedProxyMode_ToolLearnMode_ReturnsConsolidatedCommands() + { + // Arrange + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated"); + + // Act - Call a consolidated tool in learn mode + var learnParameters = new Dictionary + { + ["learn"] = true + }; + + var result = await client.CallToolAsync("get_azure_databases_details", learnParameters, cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Content); + Assert.NotEmpty(result.Content); + + // Get the text content + var textContent = result.Content.OfType().FirstOrDefault(); + Assert.NotNull(textContent); + Assert.NotEmpty(textContent.Text); + + var responseText = textContent.Text; + + // Verify the response contains information about database commands + Assert.Contains("available command", responseText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("database", responseText, StringComparison.OrdinalIgnoreCase); + + // Verify it contains multiple database-related commands + Assert.Contains("mysql", responseText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("postgres", responseText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("sql", responseText, StringComparison.OrdinalIgnoreCase); + + Output.WriteLine("Consolidated database tool learn mode response:"); + Output.WriteLine(responseText); + Output.WriteLine($"✓ Learn mode returned {responseText.Length} characters of consolidated command information"); + } + + [Fact] + public async Task ConsolidatedProxyMode_WithNamespaceFilter_LoadsFilteredConsolidatedTools() + { + // Arrange + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated", "--namespace", "storage", "--namespace", "sql"); + + // Act + var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.NotEmpty(listResult); + + var toolNames = listResult.Select(t => t.Name).ToList(); + + // Should only include consolidated tools related to specified namespaces + var hasRelevantTools = toolNames.Any(name => + name.Contains("storage", StringComparison.OrdinalIgnoreCase) || + name.Contains("sql", StringComparison.OrdinalIgnoreCase) || + name.Contains("database", StringComparison.OrdinalIgnoreCase)); + + Assert.True(hasRelevantTools, "Should have consolidated tools related to storage or database namespaces"); + + // In consolidated mode with namespace filter, should have fewer tools than without filter + Assert.True(toolNames.Count < 30, $"Expected fewer than 30 tools with namespace filter, got {toolNames.Count}"); + + Output.WriteLine($"Consolidated proxy mode with [storage, sql] namespaces loaded {toolNames.Count} tools"); + foreach (var name in toolNames) + { + Output.WriteLine($" - {name}"); + } + } + + [Fact] + public async Task ConsolidatedProxyMode_WithReadOnlyFlag_LoadsOnlyReadOnlyConsolidatedTools() + { + // Arrange + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated", "--read-only"); + + // Act + var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.NotEmpty(listResult); + + var toolCount = listResult.Count(); + Assert.True(toolCount > 0, "Should have at least some read-only consolidated tools"); + + // Verify all tools have read-only annotations + var toolsWithReadOnlyHint = 0; + var toolsWithAnnotations = 0; + + foreach (var tool in listResult) + { + var hasAnnotations = tool.ProtocolTool?.Annotations != null; + var readOnlyHint = tool.ProtocolTool?.Annotations?.ReadOnlyHint; + + if (hasAnnotations) + { + toolsWithAnnotations++; + } + + if (readOnlyHint.HasValue && readOnlyHint.Value) + { + toolsWithReadOnlyHint++; + } + + // Verify tool names don't contain destructive operations + Assert.DoesNotContain("create_", tool.Name, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("edit_", tool.Name, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("delete_", tool.Name, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("update_", tool.Name, StringComparison.OrdinalIgnoreCase); + + Output.WriteLine($"Tool: {tool.Name} - HasAnnotations: {hasAnnotations}, ReadOnlyHint: {readOnlyHint}"); + } + + Output.WriteLine($"Tools with annotations: {toolsWithAnnotations}/{toolCount}"); + Output.WriteLine($"Tools with ReadOnlyHint=true: {toolsWithReadOnlyHint}/{toolCount}"); + + // In read-only mode, ALL tools must have annotations and ReadOnlyHint=true + Assert.Equal(toolCount, toolsWithAnnotations); + Assert.Equal(toolCount, toolsWithReadOnlyHint); + + Output.WriteLine($"✓ All {toolCount} consolidated tools have annotations and ReadOnlyHint=true in read-only mode"); + } + + [Fact] + public async Task ConsolidatedProxyMode_CanCallConsolidatedTool() + { + // Arrange + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated"); + + // Act - Call the consolidated subscriptions and resource groups tool + var result = await client.CallToolAsync("get_azure_subscriptions_and_resource_groups", + new Dictionary { }, + cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Content); + Assert.NotEmpty(result.Content); + + // The result should contain subscription and resource group data + var firstContent = result.Content.FirstOrDefault(); + Assert.NotNull(firstContent); + + // Log for debugging + Output.WriteLine($"Consolidated tool result: {firstContent}"); + } + + #endregion + #region Tool Mode Tests [Fact] diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs index 8c2a94b568..aa52db4475 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs @@ -4,7 +4,6 @@ using Azure.Mcp.Core.Areas.Server.Commands.Discovery; using Azure.Mcp.Core.Areas.Server.Options; using Azure.Mcp.Core.Commands; -using Azure.Mcp.Tests.Client.Helpers; using Xunit; namespace Azure.Mcp.Core.UnitTests.Areas.Server.Commands.Discovery; @@ -27,102 +26,6 @@ private static ConsolidatedToolDiscoveryStrategy CreateStrategy( return strategy; } - [Fact] - public void Constructor_WithNullCommandFactory_DoesNotThrow() - { - // Arrange - var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); - var logger = NSubstitute.Substitute.For>(); - - // Act & Assert - // Primary constructor syntax doesn't automatically validate null parameters - var strategy = new ConsolidatedToolDiscoveryStrategy(null!, options, logger); - Assert.NotNull(strategy); - } - - [Fact] - public void Constructor_WithNullOptions_DoesNotThrow() - { - // Arrange - var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); - var logger = NSubstitute.Substitute.For>(); - - // Act & Assert - // Primary constructor syntax doesn't automatically validate null parameters - var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, null!, logger); - Assert.NotNull(strategy); - } - - [Fact] - public void Constructor_WithValidParameters_InitializesCorrectly() - { - // Arrange - var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); - var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); - var logger = NSubstitute.Substitute.For>(); - - // Act - var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger); - - // Assert - Assert.NotNull(strategy); - Assert.Null(strategy.EntryPoint); // Default should be null - } - - [Fact] - public void EntryPoint_DefaultsToNull() - { - // Arrange - var strategy = CreateStrategy(); - - // Act & Assert - Assert.Null(strategy.EntryPoint); - } - - [Fact] - public void EntryPoint_CanBeSetAndRetrieved() - { - // Arrange - var strategy = CreateStrategy(); - var azmcpPath = McpTestUtilities.GetAzMcpExecutablePath(); - - // Act - strategy.EntryPoint = azmcpPath; - - // Assert - Assert.Equal(azmcpPath, strategy.EntryPoint); - } - - [Fact] - public void EntryPoint_WhenSetToEmpty_RemainsEmpty() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - strategy.EntryPoint = ""; - - // Assert - // The strategy itself just stores the value as-is - // The defaulting behavior happens in CommandGroupServerProvider - Assert.Equal("", strategy.EntryPoint); - } - - [Fact] - public void EntryPoint_WhenSetToWhitespace_RemainsWhitespace() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - strategy.EntryPoint = " "; - - // Assert - // The strategy itself just stores the value as-is - // The defaulting behavior happens in CommandGroupServerProvider - Assert.Equal(" ", strategy.EntryPoint); - } - [Fact] public async Task DiscoverServersAsync_WithDefaultOptions_ReturnsConsolidatedToolProvider() { @@ -135,398 +38,6 @@ public async Task DiscoverServersAsync_WithDefaultOptions_ReturnsConsolidatedToo // Assert Assert.NotNull(result); var providers = result.ToList(); - Assert.Single(providers); // Should return exactly one consolidated provider - Assert.All(providers, provider => Assert.IsType(provider)); - - // Verify the provider has the expected name - var metadata = providers[0].CreateMetadata(); - Assert.Equal("get_azure_best_practices", metadata.Name); - Assert.Equal("get_azure_best_practices", metadata.Id); - } - - [Fact] - public async Task DiscoverServersAsync_WithReadOnlyFalse_CreatesNonReadOnlyProvider() - { - // Arrange - var options = new ServiceStartOptions { ReadOnly = false }; - var strategy = CreateStrategy(options: options); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - Assert.False(((CommandGroupServerProvider)providers[0]).ReadOnly); - } - - [Fact] - public async Task DiscoverServersAsync_WithReadOnlyTrue_CreatesReadOnlyProvider() - { - // Arrange - var options = new ServiceStartOptions { ReadOnly = true }; - var strategy = CreateStrategy(options: options); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - Assert.True(((CommandGroupServerProvider)providers[0]).ReadOnly); - } - - [Fact] - public async Task DiscoverServersAsync_WithNullReadOnlyOption_DefaultsToFalse() - { - // Arrange - var options = new ServiceStartOptions { ReadOnly = null }; - var strategy = CreateStrategy(options: options); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - Assert.False(((CommandGroupServerProvider)providers[0]).ReadOnly); - } - - [Fact] - public async Task DiscoverServersAsync_WithCustomEntryPoint_SetsEntryPointOnProvider() - { - // Arrange - var customEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); - var strategy = CreateStrategy(entryPoint: customEntryPoint); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - Assert.Equal(customEntryPoint, ((CommandGroupServerProvider)providers[0]).EntryPoint); - } - - [Fact] - public async Task DiscoverServersAsync_WithNullEntryPoint_UsesCurrentProcessExecutable() - { - // Arrange - var strategy = CreateStrategy(entryPoint: null); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - // When EntryPoint is set to null, CommandGroupServerProvider defaults to current process executable - var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; - var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; - Assert.NotNull(actualEntryPoint); - // Should be the current test process executable - Assert.Equal(currentProcessPath, actualEntryPoint); - } - - [Fact] - public async Task DiscoverServersAsync_WithEmptyEntryPoint_ProviderDefaultsToCurrentProcess() - { - // Arrange - var strategy = CreateStrategy(entryPoint: ""); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; - var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; - // CommandGroupServerProvider defaults empty/null to current process - Assert.Equal(currentProcessPath, actualEntryPoint); - } - - [Fact] - public async Task DiscoverServersAsync_WithWhitespaceEntryPoint_ProviderDefaultsToCurrentProcess() - { - // Arrange - var strategy = CreateStrategy(entryPoint: " "); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - var currentProcessPath = System.Diagnostics.Process.GetCurrentProcess().MainModule?.FileName; - var actualEntryPoint = ((CommandGroupServerProvider)providers[0]).EntryPoint; - // CommandGroupServerProvider defaults whitespace to current process - Assert.Equal(currentProcessPath, actualEntryPoint); - } - - [Fact] - public async Task DiscoverServersAsync_ProviderHasCorrectMetadata() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - - var metadata = providers[0].CreateMetadata(); - Assert.NotNull(metadata); - Assert.Equal("get_azure_best_practices", metadata.Name); - Assert.Equal("get_azure_best_practices", metadata.Id); - Assert.Equal(metadata.Name, metadata.Id); // Should be the same - Assert.NotNull(metadata.Description); - Assert.Contains("Azure best practices", metadata.Description); - Assert.Contains("infrastructure schema", metadata.Description); - } - - [Fact] - public async Task DiscoverServersAsync_ProviderIsCommandGroupServerProviderType() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - Assert.IsType(providers[0]); - } - - [Fact] - public async Task DiscoverServersAsync_CanBeCalledMultipleTimes() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result1 = await strategy.DiscoverServersAsync(); - var result2 = await strategy.DiscoverServersAsync(); - - // Assert - Assert.NotNull(result1); - Assert.NotNull(result2); - - var providers1 = result1.ToList(); - var providers2 = result2.ToList(); - Assert.Equal(providers1.Count, providers2.Count); - Assert.Single(providers1); - Assert.Single(providers2); - - // Should return equivalent results - var metadata1 = providers1[0].CreateMetadata(); - var metadata2 = providers2[0].CreateMetadata(); - Assert.Equal(metadata1.Name, metadata2.Name); - Assert.Equal(metadata1.Id, metadata2.Id); - Assert.Equal(metadata1.Description, metadata2.Description); - } - - [Fact] - public async Task DiscoverServersAsync_RespectsServiceStartOptionsValues() - { - // Arrange - var options = new ServiceStartOptions - { - ReadOnly = true, - }; - var azmcpEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); - var strategy = CreateStrategy(options: options, entryPoint: azmcpEntryPoint); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - - var serverProvider = (CommandGroupServerProvider)providers[0]; - Assert.True(serverProvider.ReadOnly); - Assert.Equal(azmcpEntryPoint, serverProvider.EntryPoint); - } - - [Fact] - public async Task DiscoverServersAsync_ResultCountIsConsistent() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result1 = await strategy.DiscoverServersAsync(); - var result2 = await strategy.DiscoverServersAsync(); - - // Assert - var count1 = result1.Count(); - var count2 = result2.Count(); - Assert.Equal(count1, count2); - Assert.Equal(1, count1); // Should always have exactly one consolidated provider - } - - [Fact] - public async Task DiscoverServersAsync_InheritsFromBaseDiscoveryStrategy() - { - // Arrange - var strategy = CreateStrategy(); - - // Act & Assert - Assert.IsAssignableFrom(strategy); - Assert.IsAssignableFrom(strategy); - - // Should be able to call the interface method - var result = await strategy.DiscoverServersAsync(); - Assert.NotNull(result); - } - - [Fact] - public async Task DiscoverServersAsync_FiltersCommandsByCompositeToolMapped() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - - var provider = (CommandGroupServerProvider)providers[0]; - var metadata = provider.CreateMetadata(); - Assert.Equal("get_azure_best_practices", metadata.Name); - - // The strategy should only create a provider for commands with CompositeToolMapped = "get_azure_best_practices" - // The actual filtering logic is tested implicitly by verifying we get the expected provider name - } - - [Fact] - public async Task DiscoverServersAsync_CreatesCommandGroupWithCorrectToolMetadata() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - - // We can't directly access the CommandGroup's ToolMetadata from the provider, - // but we can verify the provider was created successfully and has the expected characteristics - var provider = (CommandGroupServerProvider)providers[0]; - Assert.NotNull(provider); - - var metadata = provider.CreateMetadata(); - Assert.Equal("get_azure_best_practices", metadata.Name); - } - - // Keep original tests for backward compatibility - [Fact] - public async Task ShouldDiscoverConsolidatedToolServer() - { - var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); - var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions()); - var logger = NSubstitute.Substitute.For>(); - var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger); - var result = await strategy.DiscoverServersAsync(); - Assert.NotNull(result); - var providers = result.ToList(); - Assert.Single(providers); - } - - [Fact] - public async Task ShouldDiscoverConsolidatedToolServer_SetsPropertiesCorrectly() - { - var commandFactory = CommandFactoryHelpers.CreateCommandFactory(); - var options = Microsoft.Extensions.Options.Options.Create(new ServiceStartOptions { ReadOnly = true }); - var azmcpEntryPoint = McpTestUtilities.GetAzMcpExecutablePath(); - var logger = NSubstitute.Substitute.For>(); - var strategy = new ConsolidatedToolDiscoveryStrategy(commandFactory, options, logger) - { - EntryPoint = azmcpEntryPoint - }; - var result = (await strategy.DiscoverServersAsync()).ToList(); - Assert.Single(result); - - // Should have the consolidated tool provider with expected name - var provider = result[0]; - Assert.Equal("get_azure_best_practices", provider.CreateMetadata().Name); - - // Should set ReadOnly and EntryPoint as expected - var serverProvider = (CommandGroupServerProvider)provider; - Assert.True(serverProvider.ReadOnly); - Assert.Equal(azmcpEntryPoint, serverProvider.EntryPoint); - } - - [Fact] - public void GetAzmcpExecutablePath_ReturnsCorrectPathForCurrentOS() - { - // Arrange & Act - var azmcpPath = McpTestUtilities.GetAzMcpExecutablePath(); - - // Assert - Assert.NotNull(azmcpPath); - Assert.NotEmpty(azmcpPath); - - // Should end with the correct executable name for the current OS - if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform( - System.Runtime.InteropServices.OSPlatform.Windows)) - { - Assert.EndsWith("azmcp.exe", azmcpPath); - } - else - { - Assert.EndsWith("azmcp", azmcpPath); - Assert.False(azmcpPath.EndsWith("azmcp.exe")); - } - - // Should be in the same directory as the test assembly - var testAssemblyPath = System.Reflection.Assembly.GetExecutingAssembly().Location; - var testDirectory = Path.GetDirectoryName(testAssemblyPath); - var expectedDirectory = Path.GetDirectoryName(azmcpPath); - Assert.Equal(testDirectory, expectedDirectory); - } - - [Fact] - public async Task DiscoverServersAsync_AlwaysReturnsOneProvider() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - // Unlike CommandGroupDiscoveryStrategy which returns multiple providers (one per command group), - // ConsolidatedToolDiscoveryStrategy always returns exactly one provider that consolidates - // all commands with the same CompositeToolMapped value - Assert.Single(providers); - } - - [Fact] - public async Task DiscoverServersAsync_ProviderNameMatchesCompositeToolMappedValue() - { - // Arrange - var strategy = CreateStrategy(); - - // Act - var result = await strategy.DiscoverServersAsync(); - - // Assert - var providers = result.ToList(); - Assert.Single(providers); - - var metadata = providers[0].CreateMetadata(); - // The provider name should match the CompositeToolMapped value used in the filtering logic - Assert.Equal("get_azure_best_practices", metadata.Name); + Assert.All(providers, provider => Assert.IsType(provider)); } } \ No newline at end of file diff --git a/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs b/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs index d9845fd90f..1f448966e2 100644 --- a/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs +++ b/tools/Azure.Mcp.Tools.AzureBestPractices/src/AzureBestPracticesSetup.cs @@ -10,7 +10,7 @@ namespace Azure.Mcp.Tools.AzureBestPractices; public class AzureBestPracticesSetup : IAreaSetup { - public string Name => "get-bestpractices"; + public string Name => "get_bestpractices"; public void ConfigureServices(IServiceCollection services) { diff --git a/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs b/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs index 01656b7b88..48ffe3a82f 100644 --- a/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs +++ b/tools/Azure.Mcp.Tools.AzureBestPractices/src/Commands/BestPracticesCommand.cs @@ -41,8 +41,6 @@ public sealed class BestPracticesCommand(ILogger logger) : Secret = false }; - public override string CompositeToolMapped => "get_azure_best_practices"; - protected override void RegisterOptions(Command command) { command.Options.Add(BestPracticesOptionDefinitions.Resource); diff --git a/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs b/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs index f23a80529c..9b796420b8 100644 --- a/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs +++ b/tools/Azure.Mcp.Tools.AzureTerraformBestPractices/src/Commands/AzureTerraformBestPracticesGetCommand.cs @@ -43,7 +43,6 @@ private static string LoadBestPracticesText() Secret = false }; - public override string CompositeToolMapped => "get_azure_best_practices"; protected override EmptyOptions BindOptions(ParseResult parseResult) => new(); public override Task ExecuteAsync(CommandContext context, ParseResult parseResult) diff --git a/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs b/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs index 25f5183a5b..c8ba216aec 100644 --- a/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs +++ b/tools/Azure.Mcp.Tools.BicepSchema/src/Commands/BicepSchemaGetCommand.cs @@ -34,8 +34,6 @@ public sealed class BicepSchemaGetCommand(ILogger logger) Secret = false }; - public override string CompositeToolMapped => "get_azure_best_practices"; - private static readonly Lazy s_serviceProvider; static BicepSchemaGetCommand() diff --git a/tools/Azure.Mcp.Tools.KeyVault/src/Commands/Admin/AdminSettingsGetCommand.cs b/tools/Azure.Mcp.Tools.KeyVault/src/Commands/Admin/AdminSettingsGetCommand.cs index 3a323117fd..08fc6bc623 100644 --- a/tools/Azure.Mcp.Tools.KeyVault/src/Commands/Admin/AdminSettingsGetCommand.cs +++ b/tools/Azure.Mcp.Tools.KeyVault/src/Commands/Admin/AdminSettingsGetCommand.cs @@ -20,7 +20,7 @@ public sealed class AdminSettingsGetCommand(ILogger log public override string Title => CommandTitle; public override ToolMetadata Metadata => new() { - OpenWorld = true, // Command queries Azure resources (vault settings) + OpenWorld = false, // Command queries Azure resources (vault settings) Destructive = false, // Command only reads settings, no modifications Idempotent = true, // Same call produces same result ReadOnly = true, // Only reads data, no state changes diff --git a/tools/Azure.Mcp.Tools.ManagedLustre/src/Commands/FileSystem/FileSystemCreateCommand.cs b/tools/Azure.Mcp.Tools.ManagedLustre/src/Commands/FileSystem/FileSystemCreateCommand.cs index 37c641d4c7..a64ec82d9e 100644 --- a/tools/Azure.Mcp.Tools.ManagedLustre/src/Commands/FileSystem/FileSystemCreateCommand.cs +++ b/tools/Azure.Mcp.Tools.ManagedLustre/src/Commands/FileSystem/FileSystemCreateCommand.cs @@ -30,7 +30,7 @@ public sealed class FileSystemCreateCommand(ILogger log public override ToolMetadata Metadata => new() { - Destructive = false, + Destructive = true, Idempotent = false, OpenWorld = false, ReadOnly = false, diff --git a/tools/Azure.Mcp.Tools.Sql/src/Commands/Server/ServerListCommand.cs b/tools/Azure.Mcp.Tools.Sql/src/Commands/Server/ServerListCommand.cs index 92a6a0bbd2..de7cc81334 100644 --- a/tools/Azure.Mcp.Tools.Sql/src/Commands/Server/ServerListCommand.cs +++ b/tools/Azure.Mcp.Tools.Sql/src/Commands/Server/ServerListCommand.cs @@ -37,7 +37,7 @@ public sealed class ServerListCommand(ILogger logger) { Destructive = false, Idempotent = true, - OpenWorld = true, + OpenWorld = false, ReadOnly = true, LocalRequired = false, Secret = false From 2c69ac4c25bde0b3b4c73e22056d306ae32150ef Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Mon, 13 Oct 2025 13:15:04 -0400 Subject: [PATCH 03/16] Fix format --- .../Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs | 4 ++-- .../src/Areas/Server/Commands/Discovery/McpServerMetadata.cs | 2 +- .../src/Areas/Server/Models/ConsolidatedToolDefinition.cs | 2 +- .../Discovery/ConsolidatedToolDiscoveryStrategyTests.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 0a8461de3a..01141bb9c7 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -159,12 +159,12 @@ private static bool AreMetadataEqual(ToolMetadata? metadata1, ToolMetadata? meta { return true; } - + if (metadata1 == null || metadata2 == null) { return false; } - + return metadata1.Destructive == metadata2.Destructive && metadata1.Idempotent == metadata2.Idempotent && metadata1.OpenWorld == metadata2.OpenWorld && diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs index c3d477a2d8..2dd9257342 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using ModelContextProtocol.Client; using Azure.Mcp.Core.Commands; +using ModelContextProtocol.Client; namespace Azure.Mcp.Core.Areas.Server.Commands.Discovery; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs index 3bd6cdcad4..0a47667ec0 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs @@ -35,4 +35,4 @@ public sealed class ConsolidatedToolDefinition /// [JsonPropertyName("mappedToolList")] public List MappedToolList { get; set; } = new List(); -} \ No newline at end of file +} diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs index aa52db4475..65397504de 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs @@ -40,4 +40,4 @@ public async Task DiscoverServersAsync_WithDefaultOptions_ReturnsConsolidatedToo var providers = result.ToList(); Assert.All(providers, provider => Assert.IsType(provider)); } -} \ No newline at end of file +} From 82b327e6d78f3f594afbda5407fd8d6535b64de6 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Mon, 13 Oct 2025 13:53:37 -0400 Subject: [PATCH 04/16] Add all the new commands --- .../ConsolidatedToolDiscoveryStrategy.cs | 2 +- .../Server/Resources/consolidated-tools.json | 86 ++++++++++++++++--- .../ActivityLog/ActivityLogListCommand.cs | 2 +- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 01141bb9c7..adcf4956ae 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -107,7 +107,7 @@ public override Task> DiscoverServersAsync() $"ReadOnly={consolidatedTool.ToolMetadata?.ReadOnly}, Secret={consolidatedTool.ToolMetadata?.Secret}, " + $"LocalRequired={consolidatedTool.ToolMetadata?.LocalRequired}]"; _logger.LogError(errorMessage); - // throw new InvalidOperationException(errorMessage); + throw new InvalidOperationException(errorMessage); } commandGroup.AddCommand(commandName, command); diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json index 35a83587aa..ae1e798dcf 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -148,7 +148,7 @@ }, { "name": "get_azure_resource_and_app_health_status", - "description": "Get Azure resource and application health status, metrics, availability, and service health events. Query Log Analytics, list Log Analytics workspaces, get the current availability status of Azure resources, list service health events and incidents, list Grafana instances, view Datadog monitored resources, perform App Lens diagnostics for comprehensive application troubleshooting, and retrieve Application Insights recommendations for performance optimization.", + "description": "Get Azure resource and application health status, metrics, availability, and service health events. Query Log Analytics, list Log Analytics workspaces, list activity logs, get the current availability status of Azure resources, list service health events and incidents, list Grafana instances, view Datadog monitored resources, perform App Lens diagnostics for comprehensive application troubleshooting, retrieve Application Insights recommendations for performance optimization, and manage web tests for application monitoring.", "toolMetadata": { "destructive": false, "idempotent": true, @@ -163,12 +163,15 @@ "azmcp_grafana_list", "azmcp_datadog_monitoredresources_list", "azmcp_monitor_workspace_list", + "azmcp_monitor_activitylog_list", "azmcp_monitor_healthmodels_entity_gethealth", "azmcp_monitor_metrics_definitions", "azmcp_monitor_metrics_query", "azmcp_monitor_resource_log_query", "azmcp_monitor_table_list", "azmcp_monitor_table_type_list", + "azmcp_monitor_webtests_get", + "azmcp_monitor_webtests_list", "azmcp_monitor_workspace_log_query", "azmcp_resourcehealth_availability-status_get", "azmcp_resourcehealth_availability-status_list", @@ -176,7 +179,37 @@ ] }, { - "name": "deploy_resources_and_applications_to_azure", + "name": "create_azure_monitor_webtests", + "description": "Create Azure Monitor web tests for application availability monitoring.", + "toolMetadata": { + "destructive": true, + "idempotent": false, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_monitor_webtests_create" + ] + }, + { + "name": "update_azure_monitor_webtests", + "description": "Update Azure Monitor web tests for application availability monitoring.", + "toolMetadata": { + "destructive": true, + "idempotent": true, + "openWorld": false, + "readOnly": false, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_monitor_webtests_update" + ] + }, + { + "name": "deploy_resources_and_applications", "description": "Deploy resources and applications to Azure. Retrieve application logs, access Bicep and Terraform rules, get CI/CD pipeline guidance, and generate deployment plans.", "toolMetadata": { "destructive": false, @@ -318,18 +351,33 @@ ] }, { - "name": "execute_azure_cli", - "description": "Answer questions about an Azure environment by executing Azure CLI commands", + "name": "generate_azure_cli_commands", + "description": "Generate Azure CLI commands from natural language descriptions to help answer questions about Azure environments and operations", "toolMetadata": { - "destructive": true, - "idempotent": false, + "destructive": false, + "idempotent": true, "openWorld": false, - "readOnly": false, + "readOnly": true, "secret": false, "localRequired": false }, "mappedToolList": [ - "azmcp_extension_az" + "azmcp_extension_cli_generate" + ] + }, + { + "name": "install_azure_cli_extensions", + "description": "Install Azure CLI extensions to extend Azure CLI functionality with additional commands and features.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": true + }, + "mappedToolList": [ + "azmcp_extension_cli_install" ] }, { @@ -513,7 +561,7 @@ }, { "name": "get_azure_ai_resources_details", - "description": "Get details about Azure AI resources including listing and querying AI Search services, listing models available to be deployed and models deployed already, knowledge index schema by AI Foundry, and listing AI Foundry agents.", + "description": "Get details about Azure AI resources including listing and querying AI Search services, listing models available to be deployed and models deployed already, knowledge index schema by AI Foundry, knowledge base and source information, and listing AI Foundry agents and resources.", "toolMetadata": { "destructive": false, "idempotent": true, @@ -526,12 +574,30 @@ "azmcp_search_service_list", "azmcp_search_index_get", "azmcp_search_index_query", + "azmcp_search_knowledge_source_get", + "azmcp_search_knowledge_base_get", "azmcp_foundry_models_deployments_list", "azmcp_foundry_models_list", "azmcp_foundry_knowledge_index_list", "azmcp_foundry_knowledge_index_schema", "azmcp_foundry_openai_models-list", - "azmcp_foundry_agents_list" + "azmcp_foundry_agents_list", + "azmcp_foundry_resource_get" + ] + }, + { + "name": "retrieve_azure_ai_knowledge_base_content", + "description": "Retrieve content from Azure AI Search knowledge bases using semantic search queries to find relevant information.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": true, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_search_knowledge_base_retrieve" ] }, { diff --git a/tools/Azure.Mcp.Tools.Monitor/src/Commands/ActivityLog/ActivityLogListCommand.cs b/tools/Azure.Mcp.Tools.Monitor/src/Commands/ActivityLog/ActivityLogListCommand.cs index f5f550016b..692f7e3eec 100644 --- a/tools/Azure.Mcp.Tools.Monitor/src/Commands/ActivityLog/ActivityLogListCommand.cs +++ b/tools/Azure.Mcp.Tools.Monitor/src/Commands/ActivityLog/ActivityLogListCommand.cs @@ -34,7 +34,7 @@ internal record ActivityLogListCommandResult(List Activity public override ToolMetadata Metadata => new() { Destructive = false, - OpenWorld = true, + OpenWorld = false, Idempotent = true, ReadOnly = true, Secret = false, From 71f3a6b9a7f06bdbe13c341029bcb67f74ad210d Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Mon, 13 Oct 2025 15:52:32 -0400 Subject: [PATCH 05/16] Merged a couple consolidated tools --- .../ConsolidatedToolDiscoveryStrategy.cs | 2 +- .../Server/Resources/consolidated-tools.json | 68 +++++++++++-------- .../ConsolidatedToolDiscoveryStrategyTests.cs | 2 +- .../src/Commands/Email/EmailSendCommand.cs | 4 +- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index adcf4956ae..9a5a09b7cf 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -18,7 +18,7 @@ namespace Azure.Mcp.Core.Areas.Server.Commands.Discovery; /// The command factory used to access available command groups. /// Options for configuring the service behavior. /// Logger instance for this discovery strategy. -public sealed class ConsolidatedToolDiscoveryStrategy(CommandFactory commandFactory, IOptions options, ILogger logger) : BaseDiscoveryStrategy(logger) +public sealed class ConsolidatedToolDiscoveryStrategy(CommandFactory commandFactory, IOptions options, ILogger logger) : BaseDiscoveryStrategy(logger) { private readonly CommandFactory _commandFactory = commandFactory; private readonly IOptions _options = options; diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json index ae1e798dcf..06a48cf44e 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -209,7 +209,7 @@ ] }, { - "name": "deploy_resources_and_applications", + "name": "deploy_azure_resources_and_applications", "description": "Deploy resources and applications to Azure. Retrieve application logs, access Bicep and Terraform rules, get CI/CD pipeline guidance, and generate deployment plans.", "toolMetadata": { "destructive": false, @@ -227,8 +227,8 @@ ] }, { - "name": "execute_azure_developer_cli", - "description": "Execute Azure Developer CLI (azd) commands for modern cloud-native application development workflows. Supports azd operations including project initialization, environment provisioning, application deployment, monitoring setup, and cleanup. Essential for cloud-native development teams using azd templates and Infrastructure as Code patterns with integrated CI/CD pipeline automation.", + "name": "execute_azure_deployments", + "description": "Execute Azure deployments for applications and AI models. Run Azure Developer CLI (azd) commands for cloud-native application development workflows including project initialization, environment provisioning, application deployment, monitoring setup, and cleanup. Deploy AI models to Azure AI Foundry. Essential for cloud-native development teams using azd templates and Infrastructure as Code patterns with integrated CI/CD pipeline automation.", "toolMetadata": { "destructive": true, "idempotent": false, @@ -238,7 +238,8 @@ "localRequired": false }, "mappedToolList": [ - "azmcp_extension_azd" + "azmcp_extension_azd", + "azmcp_foundry_models_deploy" ] }, { @@ -617,21 +618,6 @@ "azmcp_foundry_openai_chat-completions-create" ] }, - { - "name": "deploy_azure_ai_models", - "description": "Deploy a model to Azure AI Foundry.", - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": false, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "mappedToolList": [ - "azmcp_foundry_models_deploy" - ] - }, { "name": "connect_azure_ai_foundry_agents", "description": "Connect to Azure AI Foundry agents for establishing agent connections and communication channels.", @@ -798,7 +784,7 @@ }, { "name": "get_azure_messaging_service_details", - "description": "Get details about Azure messaging services including Service Bus queues, topics, subscriptions, and Event Grid topics and subscriptions.", + "description": "Get details about Azure messaging services including Service Bus queues, topics, subscriptions, Event Grid topics and subscriptions, and Event Hubs namespaces, event hubs, consumer groups, and streaming event ingestion resources.", "toolMetadata": { "destructive": false, "idempotent": true, @@ -812,22 +798,30 @@ "azmcp_eventgrid_subscription_list", "azmcp_servicebus_queue_details", "azmcp_servicebus_topic_details", - "azmcp_servicebus_topic_subscription_details" + "azmcp_servicebus_topic_subscription_details", + "azmcp_eventhubs_namespace_get", + "azmcp_eventhubs_eventhub_get", + "azmcp_eventhubs_eventhub_consumergroup_get" ] }, { - "name": "get_azure_data_analytics_details", - "description": "Get details about Azure Event Hubs namespaces and streaming event ingestion resources.", + "name": "edit_azure_data_analytics_resources", + "description": "Update and delete Azure Event Hubs resources including namespaces, event hubs, and consumer groups.", "toolMetadata": { - "destructive": false, + "destructive": true, "idempotent": true, "openWorld": false, - "readOnly": true, + "readOnly": false, "secret": false, "localRequired": false }, "mappedToolList": [ - "azmcp_eventhubs_namespace_get" + "azmcp_eventhubs_namespace_update", + "azmcp_eventhubs_namespace_delete", + "azmcp_eventhubs_eventhub_update", + "azmcp_eventhubs_eventhub_delete", + "azmcp_eventhubs_eventhub_consumergroup_update", + "azmcp_eventhubs_eventhub_consumergroup_delete" ] }, { @@ -965,6 +959,21 @@ "azmcp_signalr_runtime_get" ] }, + { + "name": "get_azure_confidential_ledger_entries", + "description": "Retrieve tamper-proof entries from Azure Confidential Ledger instances.", + "toolMetadata": { + "destructive": false, + "idempotent": true, + "openWorld": false, + "readOnly": true, + "secret": false, + "localRequired": false + }, + "mappedToolList": [ + "azmcp_confidentialledger_entries_get" + ] + }, { "name": "append_azure_confidential_ledger_entries", "description": "Append tamper-proof entries to Azure Confidential Ledger instances and retrieve transaction identifiers.", @@ -981,8 +990,8 @@ ] }, { - "name": "send_azure_communication_sms", - "description": "Send SMS messages to one or more recipients using Azure Communication Services with delivery status tracking.", + "name": "send_azure_communication_messages", + "description": "Send SMS and email messages to one or more recipients using Azure Communication Services with delivery status tracking.", "toolMetadata": { "destructive": false, "idempotent": true, @@ -992,7 +1001,8 @@ "localRequired": false }, "mappedToolList": [ - "azmcp_communication_sms_send" + "azmcp_communication_sms_send", + "azmcp_communication_email_send" ] }, { diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs index 65397504de..319022b657 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategyTests.cs @@ -17,7 +17,7 @@ private static ConsolidatedToolDiscoveryStrategy CreateStrategy( { var factory = commandFactory ?? CommandFactoryHelpers.CreateCommandFactory(); var startOptions = Microsoft.Extensions.Options.Options.Create(options ?? new ServiceStartOptions()); - var logger = NSubstitute.Substitute.For>(); + var logger = NSubstitute.Substitute.For>(); var strategy = new ConsolidatedToolDiscoveryStrategy(factory, startOptions, logger); if (entryPoint != null) { diff --git a/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs b/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs index d509704331..4622a8b0f6 100644 --- a/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs +++ b/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs @@ -38,9 +38,9 @@ Supports HTML content and CC/BCC recipients. public override ToolMetadata Metadata => new() { Destructive = false, - ReadOnly = false, + ReadOnly = true, OpenWorld = true, - Idempotent = false, + Idempotent = true, Secret = false, LocalRequired = false }; From f896a427cd44882d456e700c99225ed0d07c94ed Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Mon, 13 Oct 2025 16:54:21 -0400 Subject: [PATCH 06/16] Add ToolDescriptionEvaluator result --- .../ConsolidatedToolDiscoveryStrategy.cs | 15 +- .../Generate-GroupedPromptsJson.ps1 | 18 +- .../Models/McpModels.cs | 4 +- eng/tools/ToolDescriptionEvaluator/Program.cs | 4 +- .../consolidated-prompts.json | 211 +- .../consolidated-tools.json | 1528 ---- .../results-consolidated.md | 7008 +++++++++++------ 7 files changed, 4835 insertions(+), 3953 deletions(-) delete mode 100644 eng/tools/ToolDescriptionEvaluator/consolidated-tools.json diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 9a5a09b7cf..3826d456bd 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -106,8 +106,12 @@ public override Task> DiscoverServersAsync() $"Idempotent={consolidatedTool.ToolMetadata?.Idempotent}, OpenWorld={consolidatedTool.ToolMetadata?.OpenWorld}, " + $"ReadOnly={consolidatedTool.ToolMetadata?.ReadOnly}, Secret={consolidatedTool.ToolMetadata?.Secret}, " + $"LocalRequired={consolidatedTool.ToolMetadata?.LocalRequired}]"; +#if DEBUG _logger.LogError(errorMessage); throw new InvalidOperationException(errorMessage); +#else + _logger.LogWarning(errorMessage); +#endif } commandGroup.AddCommand(commandName, command); @@ -126,23 +130,18 @@ public override Task> DiscoverServersAsync() providers.Add(serverProvider); } -#if DEBUG - // In debug mode, throw an error if there are unmatched commands + // Check for unmatched commands if (unmatchedCommands.Count > 0) { var unmatchedList = string.Join(", ", unmatchedCommands.OrderBy(c => c)); var errorMessage = $"Found {unmatchedCommands.Count} unmatched commands: {unmatchedList}"; +#if DEBUG _logger.LogError(errorMessage); throw new InvalidOperationException(errorMessage); - } #else - // In release mode, just log a warning - if (unmatchedCommands.Count > 0) - { - var unmatchedList = string.Join(", ", unmatchedCommands.OrderBy(c => c)); _logger.LogWarning("Found {Count} unmatched commands: {Commands}", unmatchedCommands.Count, unmatchedList); - } #endif + } return Task.FromResult>(providers); } diff --git a/eng/tools/ToolDescriptionEvaluator/Generate-GroupedPromptsJson.ps1 b/eng/tools/ToolDescriptionEvaluator/Generate-GroupedPromptsJson.ps1 index 1d49a2a9f2..418c6623b9 100644 --- a/eng/tools/ToolDescriptionEvaluator/Generate-GroupedPromptsJson.ps1 +++ b/eng/tools/ToolDescriptionEvaluator/Generate-GroupedPromptsJson.ps1 @@ -9,7 +9,7 @@ Modes: Both - Produces both outputs in a single run. Reads: - - consolidated-tools.json (contains consolidated_azure_tools array; each tool has an available_commands list) + - consolidated-tools.json (contains consolidated_tools array; each tool has an mappedToolList list) - namespace-tools.json (contains top-level namespaces with recursive subcommands) - tools.json (optional for future enrichment / validation) - prompts.json (maps individual command keys to prompt examples; e.g. command "azmcp acr registry list" => key "azmcp_acr_registry_list") @@ -45,17 +45,17 @@ Overwrite existing output file(s). Emit detailed warnings for unmatched commands. .EXAMPLES -pwsh ./Generate-ConsolidatedPrompts.ps1 -Mode Consolidated # Consolidated only -pwsh ./Generate-ConsolidatedPrompts.ps1 -Mode Namespace # Namespace only -pwsh ./Generate-ConsolidatedPrompts.ps1 -Mode Both # Both files, overwrite if exist -pwsh ./Generate-ConsolidatedPrompts.ps1 -Mode Namespace -NamespaceToolsPath ./namespace-tools.json -OutputPath ./namespace-prompts.json +pwsh ./Generate-GroupedPromptsJson.ps1 -Mode Consolidated # Consolidated only +pwsh ./Generate-GroupedPromptsJson.ps1 -Mode Namespace # Namespace only +pwsh ./Generate-GroupedPromptsJson.ps1 -Mode Both # Both files, overwrite if exist +pwsh ./Generate-GroupedPromptsJson.ps1 -Mode Namespace -NamespaceToolsPath ./namespace-tools.json -OutputPath ./namespace-prompts.json .NOTES Idempotent. Safe to re-run. Designed to be executed from repo root or script directory. #> param( [Parameter(Mandatory)][ValidateSet('Consolidated','Namespace','Both')][string]$Mode, - [string]$ConsolidatedToolsPath = "./consolidated-tools.json", + [string]$ConsolidatedToolsPath = "../../../core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json", [string]$NamespaceToolsPath = "./namespace-tools.json", [string]$PromptsPath = "./prompts.json", [string]$ToolsPath = "./tools.json", @@ -182,15 +182,15 @@ function Invoke-ConsolidatedGeneration { if (-not (Test-Path $ConsolidatedToolsPath)) { throw "Consolidated tools file not found: $ConsolidatedToolsPath" } $consolidatedJson = Get-Content -Raw -Path $ConsolidatedToolsPath | ConvertFrom-Json - if (-not $consolidatedJson.consolidated_azure_tools) { throw "Input consolidated tools JSON missing 'consolidated_azure_tools' array" } + if (-not $consolidatedJson.consolidated_tools) { throw "Input consolidated tools JSON missing 'consolidated_tools' array" } $warnings = @() $outputMap = [ordered]@{} - foreach ($tool in $consolidatedJson.consolidated_azure_tools) { + foreach ($tool in $consolidatedJson.consolidated_tools) { if (-not $tool.name) { continue } $toolName = $tool.name $available = @() - if ($tool.available_commands) { $available = $tool.available_commands } + if ($tool.mappedToolList) { $available = $tool.mappedToolList } $commandStrings = @() foreach ($cmdEntry in $available) { diff --git a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs index b4292467fb..ad6ce6c59f 100644 --- a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs @@ -124,8 +124,8 @@ public class ListToolsResult [JsonPropertyName("results")] public List? Tools { get; set; } - [JsonPropertyName("consolidated_azure_tools")] - public List? ConsolidatedAzureTools { get; set; } + [JsonPropertyName("consolidated_tools")] + public List? ConsolidatedTools { get; set; } [JsonPropertyName("duration")] public int? Duration { get; set; } diff --git a/eng/tools/ToolDescriptionEvaluator/Program.cs b/eng/tools/ToolDescriptionEvaluator/Program.cs index 9182f6a39a..7e9b1779c2 100644 --- a/eng/tools/ToolDescriptionEvaluator/Program.cs +++ b/eng/tools/ToolDescriptionEvaluator/Program.cs @@ -179,7 +179,7 @@ static async Task Main(string[] args) // Create vector database var db = new VectorDB(new CosineSimilarity()); var stopwatch = Stopwatch.StartNew(); - var tools = listToolsResult.Tools ?? listToolsResult.ConsolidatedAzureTools; + var tools = listToolsResult.Tools ?? listToolsResult.ConsolidatedTools; if (tools == null || tools.Count == 0) { @@ -1261,7 +1261,7 @@ private static async Task RunValidationModeAsync(string toolDir, string toolDesc } }; - var tools = listToolsResult.Tools ?? listToolsResult.ConsolidatedAzureTools; + var tools = listToolsResult.Tools ?? listToolsResult.ConsolidatedTools; if (tools == null || tools.Count == 0) { diff --git a/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json b/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json index 9f7e806103..59ebdc88a7 100644 --- a/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json +++ b/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json @@ -9,21 +9,12 @@ "What subscriptions do I have?" ], "get_azure_app_resource_details": [ - "Add a CosmosDB database to app service ", - "Add a database connection to my app service in resource group ", - "Add a MySQL database to app service ", - "Add a PostgreSQL database to app service ", - "Add database on server to app service ", - "Add database with retry policy to app service ", - "Configure a SQL Server database for app service ", - "Configure tenant for database in app service ", "Describe the function app in resource group ", "Get configuration for function app ", "Get function app status for ", "Get information about my function app in ", "List all function apps in my subscription", "Retrieve host name and status of function app ", - "Set connection string for database in app service ", "Show function app details for in ", "Show me my Azure function apps", "Show me the details for the function app ", @@ -31,6 +22,17 @@ "What function apps do I have?", "What is the status of function app ?" ], + "add_azure_app_service_database": [ + "Add a CosmosDB database to app service ", + "Add a database connection to my app service in resource group ", + "Add a MySQL database to app service ", + "Add a PostgreSQL database to app service ", + "Add database on server to app service ", + "Add database with retry policy to app service ", + "Configure a SQL Server database for app service ", + "Configure tenant for database in app service ", + "Set connection string for database in app service " + ], "get_azure_databases_details": [ "Display the properties of SQL server ", "Get the configuration details for SQL server ", @@ -72,6 +74,28 @@ "Show me the tables in the PostgreSQL database in server ", "Show me the value of connection timeout in seconds in my MySQL server " ], + "create_azure_sql_databases_and_servers": [ + "Create a new Azure SQL server named in resource group ", + "Create a new database called on SQL server in resource group ", + "Create a new SQL database named in server ", + "Create a SQL database with Basic tier in server ", + "Create an Azure SQL server with name in location with admin user ", + "Set up a new SQL server called in my resource group " + ], + "rename_azure_sql_databases": [ + "Rename my Azure SQL database to on server ", + "Rename the SQL database on server to " + ], + "edit_azure_sql_databases_and_servers": [ + "Delete SQL server permanently", + "Delete the Azure SQL server from resource group ", + "Delete the database called on server ", + "Delete the SQL database from server ", + "Remove database from SQL server in resource group ", + "Remove the SQL server from my subscription", + "Scale SQL database on server to use SKU", + "Update the performance tier of SQL database on server " + ], "edit_azure_databases": [ "Enable replication for my PostgreSQL server ", "Set connection timeout to 20 seconds for my MySQL server " @@ -83,23 +107,28 @@ "Get the metric for over the last with intervals", "Get the availability status for resource ", "Investigate error rates and failed requests for Application Insights resource for the last ", + "List active service health events in my subscription", "List all available table types in the Log Analytics workspace ", "List all Azure Managed Grafana in one subscription", "List all Log Analytics workspaces in my subscription", "List all monitored resources in the Datadog resource ", + "List all service health events in my subscription", "List all tables in the Log Analytics workspace ", "List availability status for all resources in my subscription", "List code optimization recommendations across my Application Insights components", "List profiler recommendations for Application Insights in resource group ", + "List the activity logs of the last month for ", "Please help me diagnose issues with my app using app lens", "Query the metric for for the last ", "Show me all available metrics and their definitions for storage account ", + "Show me Azure service health events for subscription ", "Show me code optimization recommendations for all Application Insights resources in my subscription", "Show me my Log Analytics workspaces", "Show me performance improvement recommendations from Application Insights", + "Show me planned maintenance events for my Azure services", "Show me the available table types in the Log Analytics workspace ", "Show me the health status of all my Azure resources", - "Show me the health status of entity in the Log Analytics workspace ", + "Show me the health status of entity using the health model ", "Show me the health status of the storage account ", "Show me the Log Analytics workspaces in my subscription", "Show me the logs for the past hour for the resource in the Log Analytics workspace ", @@ -111,18 +140,24 @@ "What is the availability status of virtual machine in resource group ?", "What metric definitions are available for the Application Insights resource ", "What resources in resource group have health issues?", + "What service issues have occurred in the last 30 days?", "What's the request per second rate for my Application Insights resource over the last " ], - "deploy_resources_and_applications_to_azure": [ + "create_azure_monitor_webtests": [], + "update_azure_monitor_webtests": [], + "deploy_azure_resources_and_applications": [ "Create a plan to deploy this application to azure", "How can I create a CI/CD pipeline to deploy this app to Azure?", "Show me the log of the application deployed by azd", "Show me the rules to generate bicep scripts" ], - "execute_azure_developer_cli": [], + "execute_azure_deployments": [ + "Deploy a GPT4o instance on my resource " + ], "get_azure_app_config_settings": [ "List all App Configuration stores in my subscription", "List all key-value settings in App Configuration store ", + "List all key-value settings with key name starting with 'prod-' in App Configuration store ", "Show me my App Configuration stores", "Show me the App Configuration stores in my subscription", "Show me the key-value settings in App Configuration store ", @@ -154,17 +189,23 @@ "Provide compliance recommendations for my current Azure subscription", "Scan my Azure subscription for compliance recommendations" ], - "execute_azure_cli": [], + "generate_azure_cli_commands": [ + "Get Azure CLI command to create a Storage account with name ", + "Show me how to use Azure CLI to list all virtual machines in my subscription", + "Show me the details of the storage account with Azure CLI commands" + ], + "install_azure_cli_extensions": [], "get_azure_security_configurations": [ "List all available role assignments in my subscription", "Show me the available role assignments in my subscription" ], - "get_azure_key_vault": [ + "get_azure_key_vault_items": [ "Display the certificate details for in vault ", "Display the key details for in vault ", "Enumerate certificates in key vault ", "Enumerate keys in key vault ", "Enumerate secrets in key vault ", + "Get the account settings for my key vault ", "Get the certificate from vault ", "Get the key from vault ", "List all certificates in the key vault ", @@ -177,6 +218,7 @@ "Retrieve key metadata for in vault ", "Show certificate names in the key vault ", "Show key names in the key vault ", + "Show me the account settings for managed HSM keyvault ", "Show me the certificate in the key vault ", "Show me the certificates in the key vault ", "Show me the details of the certificate in the key vault ", @@ -187,13 +229,19 @@ "Show secrets names in the key vault ", "What certificates are in the key vault ?", "What keys are in the key vault ?", - "What secrets are in the key vault ?" + "What secrets are in the key vault ?", + "What's the value of the setting in my key vault with name " + ], + "get_azure_key_vault_secret_values": [ + "Display the secret details for in vault ", + "Get the secret from vault ", + "Retrieve secret metadata for in vault ", + "Show me the details of the secret in the key vault ", + "Show me the secret in the key vault " ], "create_azure_key_vault_items": [ - "Add a new version of secret with value in vault ", "Create a new certificate called in the key vault ", "Create a new key called with the RSA type in the key vault ", - "Create a new secret called with value in the key vault ", "Create an EC key with name in the vault ", "Create an oct key in the vault ", "Create an RSA key in the vault with name ", @@ -201,7 +249,11 @@ "Generate a key with type in vault ", "Issue a certificate in key vault ", "Provision a new key vault certificate in vault ", - "Request creation of certificate in the key vault ", + "Request creation of certificate in the key vault " + ], + "create_azure_key_vault_secrets": [ + "Add a new version of secret with value in vault ", + "Create a new secret called with value in the key vault ", "Set a secret named with value in key vault ", "Store secret value in the key vault ", "Update secret to value in the key vault " @@ -247,30 +299,63 @@ "update_azure_load_testing_configurations": [ "Update a test run display name as for the id for test in the load testing resource in resource group ." ], - "search_microsoft_docs": [], "get_azure_ai_resources_details": [ + "Get details for AI Foundry resource in resource group ", + "Get the details of knowledge base in the Azure AI Search service ", + "Get the details of knowledge source in the Azure AI Search service ", "Get the schema configuration for knowledge index ", + "List all agents in my AI Foundry project", "List all AI Foundry model deployments", "List all AI Foundry models", + "List all AI Foundry resources in my subscription", + "List all available OpenAI models in my Azure resource", "List all Cognitive Search services in my subscription", "List all indexes in the Cognitive Search service ", + "List all knowledge bases in the Azure AI Search service ", + "List all knowledge bases in the search service ", "List all knowledge indexes in my AI Foundry project", + "List all knowledge sources in the Azure AI Search service ", + "List all knowledge sources in the search service ", "Search for instances of in the index in Cognitive Search service ", "Show me all AI Foundry model deployments", "Show me my Cognitive Search services", + "Show me the AI Foundry resources in resource group ", + "Show me the available agents in my AI Foundry project", "Show me the available AI Foundry models", "Show me the Cognitive Search services in my subscription", "Show me the details of the index in Cognitive Search service ", "Show me the indexes in the Cognitive Search service ", + "Show me the knowledge base in search service ", + "Show me the knowledge bases in the Azure AI Search service ", + "Show me the knowledge bases in the search service ", "Show me the knowledge indexes in my AI Foundry project", + "Show me the knowledge source in search service ", + "Show me the knowledge sources in the Azure AI Search service ", + "Show me the knowledge sources in the search service ", + "Show me the OpenAI model deployments", "Show me the schema for knowledge index in my AI Foundry project" ], - "deploy_azure_ai_models": [ - "Deploy a GPT4o instance on my resource " + "retrieve_azure_ai_knowledge_base_content": [ + "Ask knowledge base in search service to retrieve information about ", + "Find information about using knowledge base in search service ", + "Query knowledge base in search service about ", + "Run a retrieval with knowledge base in Azure AI Search service for the query ", + "Run a retrieval with knowledge base in search service for the query ", + "Search knowledge base in Azure AI Search service for ", + "What does knowledge base in search service know about " + ], + "use_azure_openai_models": [ + "Create a chat completion with the message \"Hello, how are you today?\"", + "Create a completion with the prompt \"What is Azure?\"", + "Create vector embeddings for my text using Azure OpenAI", + "Generate embeddings for the text \"Azure OpenAI Service\"" ], "connect_azure_ai_foundry_agents": [ "Query an agent in my AI foundry project" ], + "query_and_evaluate_azure_ai_foundry_agents": [ + "Query and evaluate an agent in my AI Foundry project for task_adherence" + ], "evaluate_azure_ai_foundry_agents": [ "Evaluate the full query and response I got from my agent for task_adherence" ], @@ -282,23 +367,29 @@ "List all storage accounts in my subscription including their location and SKU", "List the Azure Managed Lustre filesystems in my resource group ", "List the Azure Managed Lustre filesystems in my subscription ", - "List the Azure Managed Lustre SKUs available in ", + "List the Azure Managed Lustre SKUs available in location ", "Show me my storage accounts with whether hierarchical namespace (HNS) is enabled", "Show me the blobs in the blob container in the storage account ", "Show me the containers in the storage account ", "Show me the details for my storage account ", "Show me the properties for blob in container in storage account ", "Show me the properties of the storage container in the storage account ", - "Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings" + "Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings", + "Tell me how many IP addresses I need for an Azure Managed Lustre filesystem of size using the SKU ", + "Validate if the network can host Azure Managed Lustre filesystem of size using the SKU " ], "create_azure_storage": [ "Create a new blob container named documents with container public access in storage account ", "Create a new storage account called testaccount123 in East US region", "Create a new storage account with Data Lake Storage Gen2 enabled", "Create a storage account with premium performance and LRS replication", + "Create an Azure Managed Lustre filesystem with name , size , SKU , and subnet for availability zone in location . Maintenance should occur on at ", "Create the container using blob public access in storage account ", "Create the storage container mycontainer in storage account " ], + "update_azure_managed_lustre_filesystems": [ + "Update the maintenance window of the Azure Managed Lustre filesystem to at " + ], "upload_azure_storage_blobs": [ "Upload file to storage blob in container in storage account " ], @@ -321,14 +412,19 @@ ], "get_azure_capacity": [ "Check usage information for in region ", - "Show me the available regions for these resource types ", - "Tell me how many IP addresses I need for of " + "Show me the available regions for these resource types " ], "get_azure_messaging_service_details": [ + "Get the details of my consumer group in my event hub , namespace , and resource group ", + "Get the details of my event hub in my namespace and resource group ", + "Get the details of my namespace in my resource group ", + "List all consumer groups in my event hub in namespace ", "List all Event Grid subscriptions in subscription ", "List all Event Grid topics in my subscription", "List all Event Grid topics in resource group in subscription ", "List all Event Grid topics in subscription ", + "List all Event Hubs in my namespace ", + "List all Event Hubs namespaces in my subscription", "List Event Grid subscriptions for subscription in location ", "List Event Grid subscriptions for topic in resource group ", "List Event Grid subscriptions for topic in subscription ", @@ -340,6 +436,22 @@ "Show me the details of service bus topic ", "Show me the Event Grid topics in my subscription" ], + "edit_azure_data_analytics_resources": [ + "Create a new consumer group in my event hub , namespace , and resource group ", + "Create a new event hub in my namespace and resource group ", + "Create an new namespace in my resource group ", + "Delete my consumer group in my event hub , namespace , and resource group ", + "Delete my event hub in my namespace and resource group ", + "Delete my namespace in my resource group ", + "Update my consumer group in my event hub , namespace , and resource group ", + "Update my event hub in my namespace and resource group ", + "Update my namespace in my resource group " + ], + "publish_azure_eventgrid_events": [ + "Publish an event to Event Grid topic using with the following data ", + "Publish event to my Event Grid topic with the following events ", + "Send an event to Event Grid topic in resource group with " + ], "get_azure_data_explorer_kusto_details": [ "List all Data Explorer clusters in my subscription", "List all databases in the Data Explorer cluster ", @@ -402,5 +514,54 @@ "List all host pools in my subscription", "List all session hosts in host pool ", "List all user sessions on session host in host pool " + ], + "get_azure_signalr_details": [ + "Describe the SignalR runtime in resource group ", + "Get information about my SignalR runtime in ", + "List all SignalRs in my subscription", + "Show all the SignalRs information in ", + "Show me the details of SignalR ", + "Show me the network information of SignalR runtime " + ], + "get_azure_confidential_ledger_entries": [ + "Get entry from Confidential Ledger for transaction on ledger ", + "Get transaction from ledger " + ], + "append_azure_confidential_ledger_entries": [ + "Append {\"hello\": \"from mcp\"} to my confidential ledger in collection ", + "Append an entry to my ledger with data {\"key\": \"value\"}", + "Create an immutable ledger entry in with content {\"audit\": \"log\"}", + "Write a tamper-proof entry to ledger containing {\"transaction\": \"data\"}", + "Write an entry to confidential ledger " + ], + "send_azure_communication_messages": [ + "Send an email from my communication service to ", + "Send an email to with subject ", + "Send an email with BCC recipients", + "Send an SMS message to saying \"Hello\"", + "Send an SMS with delivery receipt tracking", + "Send broadcast SMS to and saying \"Urgent notification\"", + "Send email to multiple recipients: , ", + "Send email with CC to and ", + "Send email with custom sender name ", + "Send email with reply-to address set to ", + "Send HTML-formatted email to with subject ", + "Send SMS from my communication service to ", + "Send SMS message with custom tracking tag \"campaign1\"", + "Send SMS to from with message \"Test message\"", + "Send SMS to multiple recipients: , ", + "Send SMS with delivery reporting enabled" + ], + "recognize_speech_from_audio": [ + "Convert speech to text from audio file using endpoint ", + "Convert speech to text with comma-separated phrase hints: \"Azure, cognitive services, API\"", + "Convert speech to text with detailed output format from audio file ", + "Convert this audio file to text using Azure Speech Services", + "Recognize speech from with phrase hints for better accuracy", + "Recognize speech from my audio file with language detection", + "Transcribe audio using multiple phrase hints: \"Azure\", \"cognitive services\", \"machine learning\"", + "Transcribe audio with raw profanity output from file ", + "Transcribe speech from audio file with profanity filtering", + "Transcribe the audio file in Spanish language" ] } \ No newline at end of file diff --git a/eng/tools/ToolDescriptionEvaluator/consolidated-tools.json b/eng/tools/ToolDescriptionEvaluator/consolidated-tools.json deleted file mode 100644 index 4ea3d410b7..0000000000 --- a/eng/tools/ToolDescriptionEvaluator/consolidated-tools.json +++ /dev/null @@ -1,1528 +0,0 @@ -{ - "consolidated_azure_tools": [ - { - "name": "get_azure_subscriptions_and_resource_groups", - "description": "Get information about Azure subscriptions and resource groups that the user has access to.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_group_list", - "mcp_azure-mcp-ser_azmcp_subscription_list" - ] - }, - { - "name": "get_azure_app_resource_details", - "description": "Get details about Azure application platform services, such as Azure Functions, and manage database integrations for App Service applications.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_appservice_database_add", - "mcp_azure-mcp-ser_azmcp_functionapp_get" - ] - }, - { - "name": "get_azure_databases_details", - "description": "Comprehensive Azure database management tool for MySQL, PostgreSQL, SQL Database, SQL Server, and Cosmos DB. List and query databases, retrieve server configurations and parameters, explore table schemas, execute database queries, manage Cosmos DB containers and items, list and view detailed information about SQL servers, and view database server details across all Azure database services.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_mysql_database_list", - "mcp_azure-mcp-ser_azmcp_mysql_database_query", - "mcp_azure-mcp-ser_azmcp_mysql_server_config_get", - "mcp_azure-mcp-ser_azmcp_mysql_server_list", - "mcp_azure-mcp-ser_azmcp_mysql_server_param_get", - "mcp_azure-mcp-ser_azmcp_mysql_table_list", - "mcp_azure-mcp-ser_azmcp_mysql_table_schema_get", - "mcp_azure-mcp-ser_azmcp_postgres_database_list", - "mcp_azure-mcp-ser_azmcp_postgres_database_query", - "mcp_azure-mcp-ser_azmcp_postgres_server_config_get", - "mcp_azure-mcp-ser_azmcp_postgres_server_list", - "mcp_azure-mcp-ser_azmcp_postgres_server_param_get", - "mcp_azure-mcp-ser_azmcp_postgres_table_list", - "mcp_azure-mcp-ser_azmcp_postgres_table_schema_get", - "mcp_azure-mcp-ser_azmcp_sql_db_list", - "mcp_azure-mcp-ser_azmcp_sql_db_show", - "mcp_azure-mcp-ser_azmcp_sql_server_list", - "mcp_azure-mcp-ser_azmcp_sql_server_show", - "mcp_azure-mcp-ser_azmcp_cosmos_account_list", - "mcp_azure-mcp-ser_azmcp_cosmos_database_container_item_query", - "mcp_azure-mcp-ser_azmcp_cosmos_database_container_list", - "mcp_azure-mcp-ser_azmcp_cosmos_database_list" - ] - }, - { - "name": "edit_azure_databases", - "description": "Edit Azure MySQL and PostgreSQL database server parameters", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_mysql_server_param_set", - "mcp_azure-mcp-ser_azmcp_postgres_server_param_set" - ] - }, - { - "name": "get_azure_resource_and_app_health_status", - "description": "Get Azure resource and application health status and metrics and availability. Query Log Analytics, list Log Analytics workspaces, get the current availability status of Azure resources, list Grafana instances, view Datadog monitored resources, perform App Lens diagnostics for comprehensive application troubleshooting, and retrieve Application Insights recommendations for performance optimization.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_applicationinsights_recommendation_list", - "mcp_azure-mcp-ser_azmcp_applens_resource_diagnose", - "mcp_azure-mcp-ser_azmcp_grafana_list", - "mcp_azure-mcp-ser_azmcp_datadog_monitoredresources_list", - "mcp_azure-mcp-ser_azmcp_monitor_workspace_list", - "mcp_azure-mcp-ser_azmcp_monitor_healthmodels_entity_gethealth", - "mcp_azure-mcp-ser_azmcp_monitor_metrics_definitions", - "mcp_azure-mcp-ser_azmcp_monitor_metrics_query", - "mcp_azure-mcp-ser_azmcp_monitor_resource_log_query", - "mcp_azure-mcp-ser_azmcp_monitor_table_list", - "mcp_azure-mcp-ser_azmcp_monitor_table_type_list", - "mcp_azure-mcp-ser_azmcp_monitor_workspace_log_query", - "mcp_azure-mcp-ser_azmcp_resourcehealth_availability-status_get", - "mcp_azure-mcp-ser_azmcp_resourcehealth_availability-status_list" - ] - }, - { - "name": "deploy_resources_and_applications_to_azure", - "description": "Deploy resources and applications to Azure. Retrieve application logs, access Bicep and Terraform rules, get CI/CD pipeline guidance, and generate deployment plans.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_deploy_app_logs_get", - "mcp_azure-mcp-ser_azmcp_deploy_iac_rules_get", - "mcp_azure-mcp-ser_azmcp_deploy_pipeline_guidance_get", - "mcp_azure-mcp-ser_azmcp_deploy_plan_get" - ] - }, - { - "name": "execute_azure_developer_cli", - "description": "Execute Azure Developer CLI (azd) commands for modern cloud-native application development workflows. Supports azd operations including project initialization, environment provisioning, application deployment, monitoring setup, and cleanup. Essential for cloud-native development teams using azd templates and Infrastructure as Code patterns with integrated CI/CD pipeline automation.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_extension_azd" - ] - }, - { - "name": "get_azure_app_config_settings", - "description": "Get details about Azure App Configuration settings", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_appconfig_account_list", - "mcp_azure-mcp-ser_azmcp_appconfig_kv_list", - "mcp_azure-mcp-ser_azmcp_appconfig_kv_show" - ] - }, - { - "name": "edit_azure_app_config_settings", - "description": "Delete or set Azure App Configuration settings with write operations.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_appconfig_kv_delete", - "mcp_azure-mcp-ser_azmcp_appconfig_kv_set" - ] - }, - { - "name": "lock_unlock_azure_app_config_settings", - "description": "Lock and unlock Azure App Configuration settings", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_appconfig_kv_lock_set" - ] - }, - { - "name": "edit_azure_workbooks", - "description": "Update or delete Azure Monitor Workbooks", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_workbooks_delete", - "mcp_azure-mcp-ser_azmcp_workbooks_update" - ] - }, - { - "name": "create_azure_workbooks", - "description": "Create new Azure Monitor Workbooks", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_workbooks_create" - ] - }, - { - "name": "get_azure_workbooks_details", - "description": "Get details about Azure Monitor Workbooks including listing workbooks and viewing specific workbook configurations.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_workbooks_list", - "mcp_azure-mcp-ser_azmcp_workbooks_show" - ] - }, - { - "name": "audit_azure_resources_compliance", - "description": "Generate compliance and security audit reports for Azure resources using Azure Quick Review (azqr).", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_extension_azqr" - ] - }, - { - "name": "execute_azure_cli", - "description": "Answer questions about an Azure environment by executing Azure CLI commands", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_extension_az" - ] - }, - { - "name": "get_azure_security_configurations", - "description": "List Azure RBAC role assignments", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_role_assignment_list" - ] - }, - { - "name": "get_azure_key_vault", - "description": "View and retrieve Azure Key Vault security artifacts, including certificates, keys (both listing and individual key details), and secrets", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_keyvault_certificate_get", - "mcp_azure-mcp-ser_azmcp_keyvault_certificate_list", - "mcp_azure-mcp-ser_azmcp_keyvault_key_get", - "mcp_azure-mcp-ser_azmcp_keyvault_key_list", - "mcp_azure-mcp-ser_azmcp_keyvault_secret_list" - ] - }, - { - "name": "create_azure_key_vault_items", - "description": "Create new security artifacts in Azure Key Vault including certificates, keys, and secrets.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": true, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_keyvault_certificate_create", - "mcp_azure-mcp-ser_azmcp_keyvault_key_create", - "mcp_azure-mcp-ser_azmcp_keyvault_secret_create" - ] - }, - { - "name": "import_azure_key_vault_certificates", - "description": "Import external certificates into Azure Key Vault", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": true - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_keyvault_certificate_import" - ] - }, - { - "name": "get_azure_best_practices", - "description": "Retrieve Azure best practices and infrastructure schema for code generation, deployment, and operations. Covers general Azure practices, Azure Functions best practices, Terraform configurations, Bicep template schemas, and deployment best practices.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": false, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_azureterraformbestpractices_get", - "mcp_azure-mcp-ser_azmcp_bicepschema_get", - "mcp_azure-mcp-ser_azmcp_get_bestpractices_get" - ] - }, - { - "name": "design_azure_architecture", - "description": "Comprehensive Azure architecture design and visualization tool. Provide cloud architecture consulting and recommend optimal Azure solutions following Well-Architected Framework principles. Also generates visual Mermaid diagrams from application topologies.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": false, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_cloudarchitect_design", - "mcp_azure-mcp-ser_azmcp_deploy_architecture_diagram_generate" - ] - }, - { - "name": "get_azure_load_testing_details", - "description": "Get Azure Load Testing test configurations, results, and test resources including listing load testing resources.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_loadtesting_testresource_list", - "mcp_azure-mcp-ser_azmcp_loadtesting_test_get", - "mcp_azure-mcp-ser_azmcp_loadtesting_testrun_get", - "mcp_azure-mcp-ser_azmcp_loadtesting_testrun_list" - ] - }, - { - "name": "create_azure_load_testing", - "description": "Create Load Testing resource or execute Azure Load Testing tests.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_loadtesting_test_create", - "mcp_azure-mcp-ser_azmcp_loadtesting_testresource_create", - "mcp_azure-mcp-ser_azmcp_loadtesting_testrun_create" - ] - }, - { - "name": "update_azure_load_testing_configurations", - "description": "Update Azure Load Testing configurations and test run settings.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_loadtesting_testrun_update" - ] - }, - { - "name": "search_microsoft_docs", - "description": "Search official Microsoft and Azure documentation for accurate, up-to-date information.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_microsoft_docs_fetch", - "mcp_azure-mcp-ser_microsoft_docs_search" - ] - }, - { - "name": "get_azure_ai_resources_details", - "description": "Get details about Azure AI resources including listing and querying AI Search services, listing models available to be deployed and models deployed already and knowledge index schema by AI Foundry and related AI infrastructure.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_search_service_list", - "mcp_azure-mcp-ser_azmcp_search_index_get", - "mcp_azure-mcp-ser_azmcp_search_index_query", - "mcp_azure-mcp-ser_azmcp_foundry_models_deployments_list", - "mcp_azure-mcp-ser_azmcp_foundry_models_list", - "mcp_azure-mcp-ser_azmcp_foundry_knowledge_index_list", - "mcp_azure-mcp-ser_azmcp_foundry_knowledge_index_schema" - ] - }, - { - "name": "deploy_azure_ai_models", - "description": "Deploy a model to Azure AI Foundry.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_foundry_models_deploy" - ] - }, - { - "name": "connect_azure_ai_foundry_agents", - "description": "Connect to Azure AI Foundry agents for establishing agent connections and communication channels.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_foundry_agents_connect" - ] - }, - { - "name": "evaluate_azure_ai_foundry_agents", - "description": "Evaluate Azure AI Foundry agents for performance assessment and testing of agent capabilities.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": false, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_foundry_agents_evaluate" - ] - }, - { - "name": "get_azure_storage_details", - "description": "Get details about Azure Storage resources including Storage accounts, blob containers, blob data, and Azure Managed Lustre filesystem information including available SKUs, but not health status.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_azuremanagedlustre_filesystem_list", - "mcp_azure-mcp-ser_azmcp_azuremanagedlustre_filesystem_sku_get", - "mcp_azure-mcp-ser_azmcp_storage_account_get", - "mcp_azure-mcp-ser_azmcp_storage_blob_container_get", - "mcp_azure-mcp-ser_azmcp_storage_blob_get" - ] - }, - { - "name": "create_azure_storage", - "description": "Create Azure Storage accounts and blob containers.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_storage_account_create", - "mcp_azure-mcp-ser_azmcp_storage_blob_container_create" - ] - }, - { - "name": "upload_azure_storage_blobs", - "description": "Upload files and data to Azure Storage blob containers.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": true - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_storage_blob_upload" - ] - }, - { - "name": "get_azure_cache_for_redis_details", - "description": "Get details about Azure Cache for Redis resources including cache instances, clusters, access policies, and database configurations. List and manage Redis caches and clusters.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_redis_cache_list", - "mcp_azure-mcp-ser_azmcp_redis_cluster_list", - "mcp_azure-mcp-ser_azmcp_redis_cache_accesspolicy_list", - "mcp_azure-mcp-ser_azmcp_redis_cluster_database_list" - ] - }, - { - "name": "browse_azure_marketplace_products", - "description": "Browse products and offers in the Azure Marketplace", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_marketplace_product_list", - "mcp_azure-mcp-ser_azmcp_marketplace_product_get" - ] - }, - { - "name": "get_azure_capacity", - "description": "Check Azure resource capacity, quotas, available regions, usage limits, and high-performance computing infrastructure", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_quota_region_availability_list", - "mcp_azure-mcp-ser_azmcp_quota_usage_check", - "mcp_azure-mcp-ser_azmcp_azuremanagedlustre_filesystem_required-subnet-size" - ] - }, - { - "name": "get_azure_messaging_service_details", - "description": "Get details about Azure messaging services including Service Bus queues, topics, subscriptions, and Event Grid topics and subscriptions.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_eventgrid_topic_list", - "mcp_azure-mcp-ser_azmcp_eventgrid_subscription_list", - "mcp_azure-mcp-ser_azmcp_servicebus_queue_details", - "mcp_azure-mcp-ser_azmcp_servicebus_topic_details", - "mcp_azure-mcp-ser_azmcp_servicebus_topic_subscription_details" - ] - }, - { - "name": "get_azure_data_explorer_kusto_details", - "description": "Get details about Azure Data Explorer (Kusto). List clusters, execute KQL queries, manage databases, explore table schemas, and get data samples.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_kusto_cluster_list", - "mcp_azure-mcp-ser_azmcp_kusto_cluster_get", - "mcp_azure-mcp-ser_azmcp_kusto_database_list", - "mcp_azure-mcp-ser_azmcp_kusto_query", - "mcp_azure-mcp-ser_azmcp_kusto_sample", - "mcp_azure-mcp-ser_azmcp_kusto_table_list", - "mcp_azure-mcp-ser_azmcp_kusto_table_schema" - ] - }, - { - "name": "create_azure_database_admin_configurations", - "description": "Create Azure SQL Server firewall rules", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": false, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_sql_server_firewall-rule_create" - ] - }, - { - "name": "delete_azure_database_admin_configurations", - "description": "Delete Azure SQL Server firewall rules", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": true, - "idempotent": true, - "openWorld": true, - "readOnly": false, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_sql_server_firewall-rule_delete" - ] - }, - { - "name": "get_azure_database_admin_configuration_details", - "description": "Get details about Azure SQL Server Administration configurations including elastic pools, Entra admin assignments, and firewall rules.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_sql_elastic-pool_list", - "mcp_azure-mcp-ser_azmcp_sql_server_entra-admin_list", - "mcp_azure-mcp-ser_azmcp_sql_server_firewall-rule_list" - ] - }, - { - "name": "get_azure_container_details", - "description": "Get details about Azure container services including Azure Container Registry (ACR) and Azure Kubernetes Service (AKS). View registries, repositories, nodepools, clusters, cluster configurations, and individual nodepool details.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_acr_registry_list", - "mcp_azure-mcp-ser_azmcp_acr_registry_repository_list", - "mcp_azure-mcp-ser_azmcp_aks_cluster_list", - "mcp_azure-mcp-ser_azmcp_aks_cluster_get", - "mcp_azure-mcp-ser_azmcp_aks_nodepool_list", - "mcp_azure-mcp-ser_azmcp_aks_nodepool_get" - ] - }, - { - "name": "get_azure_virtual_desktop_details", - "description": "Get details about Azure Virtual Desktop resources including host pools, session hosts, and user sessions.", - "parameters": { - "intent": { - "type": "string", - "description": "The intent of the azure operation to perform." - }, - "command": { - "type": "string", - "description": "The command to execute against the specified tool." - }, - "parameters": { - "type": "object", - "description": "The parameters to pass to the tool command." - }, - "learn": { - "type": "boolean", - "description": "To learn about the tool and its supported child tools and parameters.", - "default": false - } - }, - "toolMetadata": { - "destructive": false, - "idempotent": true, - "openWorld": true, - "readOnly": true, - "secret": false, - "localRequired": false - }, - "available_commands": [ - "mcp_azure-mcp-ser_azmcp_virtualdesktop_hostpool_list", - "mcp_azure-mcp-ser_azmcp_virtualdesktop_hostpool_sessionhost_list", - "mcp_azure-mcp-ser_azmcp_virtualdesktop_hostpool_sessionhost_usersession-list" - ] - } - ] -} diff --git a/eng/tools/ToolDescriptionEvaluator/results-consolidated.md b/eng/tools/ToolDescriptionEvaluator/results-consolidated.md index c014d0cc62..2c5d588e5d 100644 --- a/eng/tools/ToolDescriptionEvaluator/results-consolidated.md +++ b/eng/tools/ToolDescriptionEvaluator/results-consolidated.md @@ -1,15 +1,15 @@ # Tool Selection Analysis Setup -**Setup completed:** 2025-09-30 13:34:15 -**Tool count:** 42 -**Database setup time:** 1.6216539s +**Setup completed:** 2025-10-13 16:19:19 +**Tool count:** 60 +**Database setup time:** 1.2460379s --- # Tool Selection Analysis Results -**Analysis Date:** 2025-09-30 13:34:15 -**Tool count:** 42 +**Analysis Date:** 2025-10-13 16:19:19 +**Tool count:** 60 ## Table of Contents @@ -32,15 +32,15 @@ - [Test 17: get_azure_app_resource_details](#test-17) - [Test 18: get_azure_app_resource_details](#test-18) - [Test 19: get_azure_app_resource_details](#test-19) -- [Test 20: get_azure_app_resource_details](#test-20) -- [Test 21: get_azure_app_resource_details](#test-21) -- [Test 22: get_azure_app_resource_details](#test-22) -- [Test 23: get_azure_app_resource_details](#test-23) -- [Test 24: get_azure_app_resource_details](#test-24) -- [Test 25: get_azure_app_resource_details](#test-25) -- [Test 26: get_azure_app_resource_details](#test-26) -- [Test 27: get_azure_app_resource_details](#test-27) -- [Test 28: get_azure_app_resource_details](#test-28) +- [Test 20: add_azure_app_service_database](#test-20) +- [Test 21: add_azure_app_service_database](#test-21) +- [Test 22: add_azure_app_service_database](#test-22) +- [Test 23: add_azure_app_service_database](#test-23) +- [Test 24: add_azure_app_service_database](#test-24) +- [Test 25: add_azure_app_service_database](#test-25) +- [Test 26: add_azure_app_service_database](#test-26) +- [Test 27: add_azure_app_service_database](#test-27) +- [Test 28: add_azure_app_service_database](#test-28) - [Test 29: get_azure_databases_details](#test-29) - [Test 30: get_azure_databases_details](#test-30) - [Test 31: get_azure_databases_details](#test-31) @@ -80,24 +80,24 @@ - [Test 65: get_azure_databases_details](#test-65) - [Test 66: get_azure_databases_details](#test-66) - [Test 67: get_azure_databases_details](#test-67) -- [Test 68: edit_azure_databases](#test-68) -- [Test 69: edit_azure_databases](#test-69) -- [Test 70: get_azure_resource_and_app_health_status](#test-70) -- [Test 71: get_azure_resource_and_app_health_status](#test-71) -- [Test 72: get_azure_resource_and_app_health_status](#test-72) -- [Test 73: get_azure_resource_and_app_health_status](#test-73) -- [Test 74: get_azure_resource_and_app_health_status](#test-74) -- [Test 75: get_azure_resource_and_app_health_status](#test-75) -- [Test 76: get_azure_resource_and_app_health_status](#test-76) -- [Test 77: get_azure_resource_and_app_health_status](#test-77) -- [Test 78: get_azure_resource_and_app_health_status](#test-78) -- [Test 79: get_azure_resource_and_app_health_status](#test-79) -- [Test 80: get_azure_resource_and_app_health_status](#test-80) -- [Test 81: get_azure_resource_and_app_health_status](#test-81) -- [Test 82: get_azure_resource_and_app_health_status](#test-82) -- [Test 83: get_azure_resource_and_app_health_status](#test-83) -- [Test 84: get_azure_resource_and_app_health_status](#test-84) -- [Test 85: get_azure_resource_and_app_health_status](#test-85) +- [Test 68: create_azure_sql_databases_and_servers](#test-68) +- [Test 69: create_azure_sql_databases_and_servers](#test-69) +- [Test 70: create_azure_sql_databases_and_servers](#test-70) +- [Test 71: create_azure_sql_databases_and_servers](#test-71) +- [Test 72: create_azure_sql_databases_and_servers](#test-72) +- [Test 73: create_azure_sql_databases_and_servers](#test-73) +- [Test 74: rename_azure_sql_databases](#test-74) +- [Test 75: rename_azure_sql_databases](#test-75) +- [Test 76: edit_azure_sql_databases_and_servers](#test-76) +- [Test 77: edit_azure_sql_databases_and_servers](#test-77) +- [Test 78: edit_azure_sql_databases_and_servers](#test-78) +- [Test 79: edit_azure_sql_databases_and_servers](#test-79) +- [Test 80: edit_azure_sql_databases_and_servers](#test-80) +- [Test 81: edit_azure_sql_databases_and_servers](#test-81) +- [Test 82: edit_azure_sql_databases_and_servers](#test-82) +- [Test 83: edit_azure_sql_databases_and_servers](#test-83) +- [Test 84: edit_azure_databases](#test-84) +- [Test 85: edit_azure_databases](#test-85) - [Test 86: get_azure_resource_and_app_health_status](#test-86) - [Test 87: get_azure_resource_and_app_health_status](#test-87) - [Test 88: get_azure_resource_and_app_health_status](#test-88) @@ -117,225 +117,350 @@ - [Test 102: get_azure_resource_and_app_health_status](#test-102) - [Test 103: get_azure_resource_and_app_health_status](#test-103) - [Test 104: get_azure_resource_and_app_health_status](#test-104) -- [Test 105: deploy_resources_and_applications_to_azure](#test-105) -- [Test 106: deploy_resources_and_applications_to_azure](#test-106) -- [Test 107: deploy_resources_and_applications_to_azure](#test-107) -- [Test 108: deploy_resources_and_applications_to_azure](#test-108) -- [Test 109: get_azure_app_config_settings](#test-109) -- [Test 110: get_azure_app_config_settings](#test-110) -- [Test 111: get_azure_app_config_settings](#test-111) -- [Test 112: get_azure_app_config_settings](#test-112) -- [Test 113: get_azure_app_config_settings](#test-113) -- [Test 114: get_azure_app_config_settings](#test-114) -- [Test 115: edit_azure_app_config_settings](#test-115) -- [Test 116: edit_azure_app_config_settings](#test-116) -- [Test 117: lock_unlock_azure_app_config_settings](#test-117) -- [Test 118: lock_unlock_azure_app_config_settings](#test-118) -- [Test 119: edit_azure_workbooks](#test-119) -- [Test 120: edit_azure_workbooks](#test-120) -- [Test 121: create_azure_workbooks](#test-121) -- [Test 122: get_azure_workbooks_details](#test-122) -- [Test 123: get_azure_workbooks_details](#test-123) -- [Test 124: get_azure_workbooks_details](#test-124) -- [Test 125: get_azure_workbooks_details](#test-125) -- [Test 126: audit_azure_resources_compliance](#test-126) -- [Test 127: audit_azure_resources_compliance](#test-127) -- [Test 128: audit_azure_resources_compliance](#test-128) -- [Test 129: get_azure_security_configurations](#test-129) -- [Test 130: get_azure_security_configurations](#test-130) -- [Test 131: get_azure_key_vault](#test-131) -- [Test 132: get_azure_key_vault](#test-132) -- [Test 133: get_azure_key_vault](#test-133) -- [Test 134: get_azure_key_vault](#test-134) -- [Test 135: get_azure_key_vault](#test-135) -- [Test 136: get_azure_key_vault](#test-136) -- [Test 137: get_azure_key_vault](#test-137) -- [Test 138: get_azure_key_vault](#test-138) -- [Test 139: get_azure_key_vault](#test-139) -- [Test 140: get_azure_key_vault](#test-140) -- [Test 141: get_azure_key_vault](#test-141) -- [Test 142: get_azure_key_vault](#test-142) -- [Test 143: get_azure_key_vault](#test-143) -- [Test 144: get_azure_key_vault](#test-144) -- [Test 145: get_azure_key_vault](#test-145) -- [Test 146: get_azure_key_vault](#test-146) -- [Test 147: get_azure_key_vault](#test-147) -- [Test 148: get_azure_key_vault](#test-148) -- [Test 149: get_azure_key_vault](#test-149) -- [Test 150: get_azure_key_vault](#test-150) -- [Test 151: get_azure_key_vault](#test-151) -- [Test 152: get_azure_key_vault](#test-152) -- [Test 153: get_azure_key_vault](#test-153) -- [Test 154: get_azure_key_vault](#test-154) -- [Test 155: get_azure_key_vault](#test-155) -- [Test 156: get_azure_key_vault](#test-156) -- [Test 157: get_azure_key_vault](#test-157) -- [Test 158: get_azure_key_vault](#test-158) -- [Test 159: create_azure_key_vault_items](#test-159) -- [Test 160: create_azure_key_vault_items](#test-160) -- [Test 161: create_azure_key_vault_items](#test-161) -- [Test 162: create_azure_key_vault_items](#test-162) -- [Test 163: create_azure_key_vault_items](#test-163) -- [Test 164: create_azure_key_vault_items](#test-164) -- [Test 165: create_azure_key_vault_items](#test-165) -- [Test 166: create_azure_key_vault_items](#test-166) -- [Test 167: create_azure_key_vault_items](#test-167) -- [Test 168: create_azure_key_vault_items](#test-168) -- [Test 169: create_azure_key_vault_items](#test-169) -- [Test 170: create_azure_key_vault_items](#test-170) -- [Test 171: create_azure_key_vault_items](#test-171) -- [Test 172: create_azure_key_vault_items](#test-172) -- [Test 173: create_azure_key_vault_items](#test-173) -- [Test 174: import_azure_key_vault_certificates](#test-174) -- [Test 175: import_azure_key_vault_certificates](#test-175) -- [Test 176: import_azure_key_vault_certificates](#test-176) -- [Test 177: import_azure_key_vault_certificates](#test-177) -- [Test 178: import_azure_key_vault_certificates](#test-178) -- [Test 179: get_azure_best_practices](#test-179) -- [Test 180: get_azure_best_practices](#test-180) -- [Test 181: get_azure_best_practices](#test-181) -- [Test 182: get_azure_best_practices](#test-182) -- [Test 183: get_azure_best_practices](#test-183) -- [Test 184: get_azure_best_practices](#test-184) -- [Test 185: get_azure_best_practices](#test-185) -- [Test 186: get_azure_best_practices](#test-186) -- [Test 187: get_azure_best_practices](#test-187) -- [Test 188: get_azure_best_practices](#test-188) -- [Test 189: get_azure_best_practices](#test-189) -- [Test 190: design_azure_architecture](#test-190) -- [Test 191: design_azure_architecture](#test-191) -- [Test 192: design_azure_architecture](#test-192) -- [Test 193: design_azure_architecture](#test-193) -- [Test 194: design_azure_architecture](#test-194) -- [Test 195: get_azure_load_testing_details](#test-195) -- [Test 196: get_azure_load_testing_details](#test-196) -- [Test 197: get_azure_load_testing_details](#test-197) -- [Test 198: get_azure_load_testing_details](#test-198) -- [Test 199: create_azure_load_testing](#test-199) -- [Test 200: create_azure_load_testing](#test-200) -- [Test 201: create_azure_load_testing](#test-201) -- [Test 202: update_azure_load_testing_configurations](#test-202) -- [Test 203: get_azure_ai_resources_details](#test-203) -- [Test 204: get_azure_ai_resources_details](#test-204) -- [Test 205: get_azure_ai_resources_details](#test-205) -- [Test 206: get_azure_ai_resources_details](#test-206) -- [Test 207: get_azure_ai_resources_details](#test-207) -- [Test 208: get_azure_ai_resources_details](#test-208) -- [Test 209: get_azure_ai_resources_details](#test-209) -- [Test 210: get_azure_ai_resources_details](#test-210) -- [Test 211: get_azure_ai_resources_details](#test-211) -- [Test 212: get_azure_ai_resources_details](#test-212) -- [Test 213: get_azure_ai_resources_details](#test-213) -- [Test 214: get_azure_ai_resources_details](#test-214) -- [Test 215: get_azure_ai_resources_details](#test-215) -- [Test 216: get_azure_ai_resources_details](#test-216) -- [Test 217: get_azure_ai_resources_details](#test-217) -- [Test 218: deploy_azure_ai_models](#test-218) -- [Test 219: connect_azure_ai_foundry_agents](#test-219) -- [Test 220: evaluate_azure_ai_foundry_agents](#test-220) -- [Test 221: get_azure_storage_details](#test-221) -- [Test 222: get_azure_storage_details](#test-222) -- [Test 223: get_azure_storage_details](#test-223) -- [Test 224: get_azure_storage_details](#test-224) -- [Test 225: get_azure_storage_details](#test-225) -- [Test 226: get_azure_storage_details](#test-226) -- [Test 227: get_azure_storage_details](#test-227) -- [Test 228: get_azure_storage_details](#test-228) -- [Test 229: get_azure_storage_details](#test-229) -- [Test 230: get_azure_storage_details](#test-230) -- [Test 231: get_azure_storage_details](#test-231) -- [Test 232: get_azure_storage_details](#test-232) -- [Test 233: get_azure_storage_details](#test-233) -- [Test 234: get_azure_storage_details](#test-234) -- [Test 235: get_azure_storage_details](#test-235) -- [Test 236: create_azure_storage](#test-236) -- [Test 237: create_azure_storage](#test-237) -- [Test 238: create_azure_storage](#test-238) -- [Test 239: create_azure_storage](#test-239) -- [Test 240: create_azure_storage](#test-240) -- [Test 241: create_azure_storage](#test-241) -- [Test 242: upload_azure_storage_blobs](#test-242) -- [Test 243: get_azure_cache_for_redis_details](#test-243) -- [Test 244: get_azure_cache_for_redis_details](#test-244) -- [Test 245: get_azure_cache_for_redis_details](#test-245) -- [Test 246: get_azure_cache_for_redis_details](#test-246) -- [Test 247: get_azure_cache_for_redis_details](#test-247) -- [Test 248: get_azure_cache_for_redis_details](#test-248) -- [Test 249: get_azure_cache_for_redis_details](#test-249) -- [Test 250: get_azure_cache_for_redis_details](#test-250) -- [Test 251: get_azure_cache_for_redis_details](#test-251) -- [Test 252: get_azure_cache_for_redis_details](#test-252) -- [Test 253: browse_azure_marketplace_products](#test-253) -- [Test 254: browse_azure_marketplace_products](#test-254) -- [Test 255: browse_azure_marketplace_products](#test-255) -- [Test 256: get_azure_capacity](#test-256) -- [Test 257: get_azure_capacity](#test-257) -- [Test 258: get_azure_capacity](#test-258) -- [Test 259: get_azure_messaging_service_details](#test-259) -- [Test 260: get_azure_messaging_service_details](#test-260) -- [Test 261: get_azure_messaging_service_details](#test-261) -- [Test 262: get_azure_messaging_service_details](#test-262) -- [Test 263: get_azure_messaging_service_details](#test-263) -- [Test 264: get_azure_messaging_service_details](#test-264) -- [Test 265: get_azure_messaging_service_details](#test-265) -- [Test 266: get_azure_messaging_service_details](#test-266) -- [Test 267: get_azure_messaging_service_details](#test-267) -- [Test 268: get_azure_messaging_service_details](#test-268) -- [Test 269: get_azure_messaging_service_details](#test-269) -- [Test 270: get_azure_messaging_service_details](#test-270) -- [Test 271: get_azure_messaging_service_details](#test-271) -- [Test 272: get_azure_messaging_service_details](#test-272) -- [Test 273: get_azure_data_explorer_kusto_details](#test-273) -- [Test 274: get_azure_data_explorer_kusto_details](#test-274) -- [Test 275: get_azure_data_explorer_kusto_details](#test-275) -- [Test 276: get_azure_data_explorer_kusto_details](#test-276) -- [Test 277: get_azure_data_explorer_kusto_details](#test-277) -- [Test 278: get_azure_data_explorer_kusto_details](#test-278) -- [Test 279: get_azure_data_explorer_kusto_details](#test-279) -- [Test 280: get_azure_data_explorer_kusto_details](#test-280) -- [Test 281: get_azure_data_explorer_kusto_details](#test-281) -- [Test 282: get_azure_data_explorer_kusto_details](#test-282) -- [Test 283: get_azure_data_explorer_kusto_details](#test-283) -- [Test 284: create_azure_database_admin_configurations](#test-284) -- [Test 285: create_azure_database_admin_configurations](#test-285) -- [Test 286: create_azure_database_admin_configurations](#test-286) -- [Test 287: delete_azure_database_admin_configurations](#test-287) -- [Test 288: delete_azure_database_admin_configurations](#test-288) -- [Test 289: delete_azure_database_admin_configurations](#test-289) -- [Test 290: get_azure_database_admin_configuration_details](#test-290) -- [Test 291: get_azure_database_admin_configuration_details](#test-291) -- [Test 292: get_azure_database_admin_configuration_details](#test-292) -- [Test 293: get_azure_database_admin_configuration_details](#test-293) -- [Test 294: get_azure_database_admin_configuration_details](#test-294) -- [Test 295: get_azure_database_admin_configuration_details](#test-295) -- [Test 296: get_azure_database_admin_configuration_details](#test-296) -- [Test 297: get_azure_database_admin_configuration_details](#test-297) -- [Test 298: get_azure_database_admin_configuration_details](#test-298) -- [Test 299: get_azure_container_details](#test-299) -- [Test 300: get_azure_container_details](#test-300) -- [Test 301: get_azure_container_details](#test-301) -- [Test 302: get_azure_container_details](#test-302) -- [Test 303: get_azure_container_details](#test-303) -- [Test 304: get_azure_container_details](#test-304) -- [Test 305: get_azure_container_details](#test-305) -- [Test 306: get_azure_container_details](#test-306) -- [Test 307: get_azure_container_details](#test-307) -- [Test 308: get_azure_container_details](#test-308) -- [Test 309: get_azure_container_details](#test-309) -- [Test 310: get_azure_container_details](#test-310) -- [Test 311: get_azure_container_details](#test-311) -- [Test 312: get_azure_container_details](#test-312) -- [Test 313: get_azure_container_details](#test-313) -- [Test 314: get_azure_container_details](#test-314) -- [Test 315: get_azure_container_details](#test-315) -- [Test 316: get_azure_container_details](#test-316) -- [Test 317: get_azure_container_details](#test-317) -- [Test 318: get_azure_container_details](#test-318) -- [Test 319: get_azure_container_details](#test-319) -- [Test 320: get_azure_container_details](#test-320) -- [Test 321: get_azure_virtual_desktop_details](#test-321) -- [Test 322: get_azure_virtual_desktop_details](#test-322) -- [Test 323: get_azure_virtual_desktop_details](#test-323) +- [Test 105: get_azure_resource_and_app_health_status](#test-105) +- [Test 106: get_azure_resource_and_app_health_status](#test-106) +- [Test 107: get_azure_resource_and_app_health_status](#test-107) +- [Test 108: get_azure_resource_and_app_health_status](#test-108) +- [Test 109: get_azure_resource_and_app_health_status](#test-109) +- [Test 110: get_azure_resource_and_app_health_status](#test-110) +- [Test 111: get_azure_resource_and_app_health_status](#test-111) +- [Test 112: get_azure_resource_and_app_health_status](#test-112) +- [Test 113: get_azure_resource_and_app_health_status](#test-113) +- [Test 114: get_azure_resource_and_app_health_status](#test-114) +- [Test 115: get_azure_resource_and_app_health_status](#test-115) +- [Test 116: get_azure_resource_and_app_health_status](#test-116) +- [Test 117: get_azure_resource_and_app_health_status](#test-117) +- [Test 118: get_azure_resource_and_app_health_status](#test-118) +- [Test 119: get_azure_resource_and_app_health_status](#test-119) +- [Test 120: get_azure_resource_and_app_health_status](#test-120) +- [Test 121: get_azure_resource_and_app_health_status](#test-121) +- [Test 122: get_azure_resource_and_app_health_status](#test-122) +- [Test 123: get_azure_resource_and_app_health_status](#test-123) +- [Test 124: get_azure_resource_and_app_health_status](#test-124) +- [Test 125: get_azure_resource_and_app_health_status](#test-125) +- [Test 126: get_azure_resource_and_app_health_status](#test-126) +- [Test 127: deploy_azure_resources_and_applications](#test-127) +- [Test 128: deploy_azure_resources_and_applications](#test-128) +- [Test 129: deploy_azure_resources_and_applications](#test-129) +- [Test 130: deploy_azure_resources_and_applications](#test-130) +- [Test 131: execute_azure_deployments](#test-131) +- [Test 132: get_azure_app_config_settings](#test-132) +- [Test 133: get_azure_app_config_settings](#test-133) +- [Test 134: get_azure_app_config_settings](#test-134) +- [Test 135: get_azure_app_config_settings](#test-135) +- [Test 136: get_azure_app_config_settings](#test-136) +- [Test 137: get_azure_app_config_settings](#test-137) +- [Test 138: get_azure_app_config_settings](#test-138) +- [Test 139: edit_azure_app_config_settings](#test-139) +- [Test 140: edit_azure_app_config_settings](#test-140) +- [Test 141: lock_unlock_azure_app_config_settings](#test-141) +- [Test 142: lock_unlock_azure_app_config_settings](#test-142) +- [Test 143: edit_azure_workbooks](#test-143) +- [Test 144: edit_azure_workbooks](#test-144) +- [Test 145: create_azure_workbooks](#test-145) +- [Test 146: get_azure_workbooks_details](#test-146) +- [Test 147: get_azure_workbooks_details](#test-147) +- [Test 148: get_azure_workbooks_details](#test-148) +- [Test 149: get_azure_workbooks_details](#test-149) +- [Test 150: audit_azure_resources_compliance](#test-150) +- [Test 151: audit_azure_resources_compliance](#test-151) +- [Test 152: audit_azure_resources_compliance](#test-152) +- [Test 153: generate_azure_cli_commands](#test-153) +- [Test 154: generate_azure_cli_commands](#test-154) +- [Test 155: generate_azure_cli_commands](#test-155) +- [Test 156: get_azure_security_configurations](#test-156) +- [Test 157: get_azure_security_configurations](#test-157) +- [Test 158: get_azure_key_vault_items](#test-158) +- [Test 159: get_azure_key_vault_items](#test-159) +- [Test 160: get_azure_key_vault_items](#test-160) +- [Test 161: get_azure_key_vault_items](#test-161) +- [Test 162: get_azure_key_vault_items](#test-162) +- [Test 163: get_azure_key_vault_items](#test-163) +- [Test 164: get_azure_key_vault_items](#test-164) +- [Test 165: get_azure_key_vault_items](#test-165) +- [Test 166: get_azure_key_vault_items](#test-166) +- [Test 167: get_azure_key_vault_items](#test-167) +- [Test 168: get_azure_key_vault_items](#test-168) +- [Test 169: get_azure_key_vault_items](#test-169) +- [Test 170: get_azure_key_vault_items](#test-170) +- [Test 171: get_azure_key_vault_items](#test-171) +- [Test 172: get_azure_key_vault_items](#test-172) +- [Test 173: get_azure_key_vault_items](#test-173) +- [Test 174: get_azure_key_vault_items](#test-174) +- [Test 175: get_azure_key_vault_items](#test-175) +- [Test 176: get_azure_key_vault_items](#test-176) +- [Test 177: get_azure_key_vault_items](#test-177) +- [Test 178: get_azure_key_vault_items](#test-178) +- [Test 179: get_azure_key_vault_items](#test-179) +- [Test 180: get_azure_key_vault_items](#test-180) +- [Test 181: get_azure_key_vault_items](#test-181) +- [Test 182: get_azure_key_vault_items](#test-182) +- [Test 183: get_azure_key_vault_items](#test-183) +- [Test 184: get_azure_key_vault_items](#test-184) +- [Test 185: get_azure_key_vault_items](#test-185) +- [Test 186: get_azure_key_vault_items](#test-186) +- [Test 187: get_azure_key_vault_items](#test-187) +- [Test 188: get_azure_key_vault_items](#test-188) +- [Test 189: get_azure_key_vault_secret_values](#test-189) +- [Test 190: get_azure_key_vault_secret_values](#test-190) +- [Test 191: get_azure_key_vault_secret_values](#test-191) +- [Test 192: get_azure_key_vault_secret_values](#test-192) +- [Test 193: get_azure_key_vault_secret_values](#test-193) +- [Test 194: create_azure_key_vault_items](#test-194) +- [Test 195: create_azure_key_vault_items](#test-195) +- [Test 196: create_azure_key_vault_items](#test-196) +- [Test 197: create_azure_key_vault_items](#test-197) +- [Test 198: create_azure_key_vault_items](#test-198) +- [Test 199: create_azure_key_vault_items](#test-199) +- [Test 200: create_azure_key_vault_items](#test-200) +- [Test 201: create_azure_key_vault_items](#test-201) +- [Test 202: create_azure_key_vault_items](#test-202) +- [Test 203: create_azure_key_vault_items](#test-203) +- [Test 204: create_azure_key_vault_secrets](#test-204) +- [Test 205: create_azure_key_vault_secrets](#test-205) +- [Test 206: create_azure_key_vault_secrets](#test-206) +- [Test 207: create_azure_key_vault_secrets](#test-207) +- [Test 208: create_azure_key_vault_secrets](#test-208) +- [Test 209: import_azure_key_vault_certificates](#test-209) +- [Test 210: import_azure_key_vault_certificates](#test-210) +- [Test 211: import_azure_key_vault_certificates](#test-211) +- [Test 212: import_azure_key_vault_certificates](#test-212) +- [Test 213: import_azure_key_vault_certificates](#test-213) +- [Test 214: get_azure_best_practices](#test-214) +- [Test 215: get_azure_best_practices](#test-215) +- [Test 216: get_azure_best_practices](#test-216) +- [Test 217: get_azure_best_practices](#test-217) +- [Test 218: get_azure_best_practices](#test-218) +- [Test 219: get_azure_best_practices](#test-219) +- [Test 220: get_azure_best_practices](#test-220) +- [Test 221: get_azure_best_practices](#test-221) +- [Test 222: get_azure_best_practices](#test-222) +- [Test 223: get_azure_best_practices](#test-223) +- [Test 224: get_azure_best_practices](#test-224) +- [Test 225: design_azure_architecture](#test-225) +- [Test 226: design_azure_architecture](#test-226) +- [Test 227: design_azure_architecture](#test-227) +- [Test 228: design_azure_architecture](#test-228) +- [Test 229: design_azure_architecture](#test-229) +- [Test 230: get_azure_load_testing_details](#test-230) +- [Test 231: get_azure_load_testing_details](#test-231) +- [Test 232: get_azure_load_testing_details](#test-232) +- [Test 233: get_azure_load_testing_details](#test-233) +- [Test 234: create_azure_load_testing](#test-234) +- [Test 235: create_azure_load_testing](#test-235) +- [Test 236: create_azure_load_testing](#test-236) +- [Test 237: update_azure_load_testing_configurations](#test-237) +- [Test 238: get_azure_ai_resources_details](#test-238) +- [Test 239: get_azure_ai_resources_details](#test-239) +- [Test 240: get_azure_ai_resources_details](#test-240) +- [Test 241: get_azure_ai_resources_details](#test-241) +- [Test 242: get_azure_ai_resources_details](#test-242) +- [Test 243: get_azure_ai_resources_details](#test-243) +- [Test 244: get_azure_ai_resources_details](#test-244) +- [Test 245: get_azure_ai_resources_details](#test-245) +- [Test 246: get_azure_ai_resources_details](#test-246) +- [Test 247: get_azure_ai_resources_details](#test-247) +- [Test 248: get_azure_ai_resources_details](#test-248) +- [Test 249: get_azure_ai_resources_details](#test-249) +- [Test 250: get_azure_ai_resources_details](#test-250) +- [Test 251: get_azure_ai_resources_details](#test-251) +- [Test 252: get_azure_ai_resources_details](#test-252) +- [Test 253: get_azure_ai_resources_details](#test-253) +- [Test 254: get_azure_ai_resources_details](#test-254) +- [Test 255: get_azure_ai_resources_details](#test-255) +- [Test 256: get_azure_ai_resources_details](#test-256) +- [Test 257: get_azure_ai_resources_details](#test-257) +- [Test 258: get_azure_ai_resources_details](#test-258) +- [Test 259: get_azure_ai_resources_details](#test-259) +- [Test 260: get_azure_ai_resources_details](#test-260) +- [Test 261: get_azure_ai_resources_details](#test-261) +- [Test 262: get_azure_ai_resources_details](#test-262) +- [Test 263: get_azure_ai_resources_details](#test-263) +- [Test 264: get_azure_ai_resources_details](#test-264) +- [Test 265: get_azure_ai_resources_details](#test-265) +- [Test 266: get_azure_ai_resources_details](#test-266) +- [Test 267: get_azure_ai_resources_details](#test-267) +- [Test 268: get_azure_ai_resources_details](#test-268) +- [Test 269: get_azure_ai_resources_details](#test-269) +- [Test 270: get_azure_ai_resources_details](#test-270) +- [Test 271: get_azure_ai_resources_details](#test-271) +- [Test 272: retrieve_azure_ai_knowledge_base_content](#test-272) +- [Test 273: retrieve_azure_ai_knowledge_base_content](#test-273) +- [Test 274: retrieve_azure_ai_knowledge_base_content](#test-274) +- [Test 275: retrieve_azure_ai_knowledge_base_content](#test-275) +- [Test 276: retrieve_azure_ai_knowledge_base_content](#test-276) +- [Test 277: retrieve_azure_ai_knowledge_base_content](#test-277) +- [Test 278: retrieve_azure_ai_knowledge_base_content](#test-278) +- [Test 279: use_azure_openai_models](#test-279) +- [Test 280: use_azure_openai_models](#test-280) +- [Test 281: use_azure_openai_models](#test-281) +- [Test 282: use_azure_openai_models](#test-282) +- [Test 283: connect_azure_ai_foundry_agents](#test-283) +- [Test 284: query_and_evaluate_azure_ai_foundry_agents](#test-284) +- [Test 285: evaluate_azure_ai_foundry_agents](#test-285) +- [Test 286: get_azure_storage_details](#test-286) +- [Test 287: get_azure_storage_details](#test-287) +- [Test 288: get_azure_storage_details](#test-288) +- [Test 289: get_azure_storage_details](#test-289) +- [Test 290: get_azure_storage_details](#test-290) +- [Test 291: get_azure_storage_details](#test-291) +- [Test 292: get_azure_storage_details](#test-292) +- [Test 293: get_azure_storage_details](#test-293) +- [Test 294: get_azure_storage_details](#test-294) +- [Test 295: get_azure_storage_details](#test-295) +- [Test 296: get_azure_storage_details](#test-296) +- [Test 297: get_azure_storage_details](#test-297) +- [Test 298: get_azure_storage_details](#test-298) +- [Test 299: get_azure_storage_details](#test-299) +- [Test 300: get_azure_storage_details](#test-300) +- [Test 301: get_azure_storage_details](#test-301) +- [Test 302: get_azure_storage_details](#test-302) +- [Test 303: create_azure_storage](#test-303) +- [Test 304: create_azure_storage](#test-304) +- [Test 305: create_azure_storage](#test-305) +- [Test 306: create_azure_storage](#test-306) +- [Test 307: create_azure_storage](#test-307) +- [Test 308: create_azure_storage](#test-308) +- [Test 309: create_azure_storage](#test-309) +- [Test 310: update_azure_managed_lustre_filesystems](#test-310) +- [Test 311: upload_azure_storage_blobs](#test-311) +- [Test 312: get_azure_cache_for_redis_details](#test-312) +- [Test 313: get_azure_cache_for_redis_details](#test-313) +- [Test 314: get_azure_cache_for_redis_details](#test-314) +- [Test 315: get_azure_cache_for_redis_details](#test-315) +- [Test 316: get_azure_cache_for_redis_details](#test-316) +- [Test 317: get_azure_cache_for_redis_details](#test-317) +- [Test 318: get_azure_cache_for_redis_details](#test-318) +- [Test 319: get_azure_cache_for_redis_details](#test-319) +- [Test 320: get_azure_cache_for_redis_details](#test-320) +- [Test 321: get_azure_cache_for_redis_details](#test-321) +- [Test 322: browse_azure_marketplace_products](#test-322) +- [Test 323: browse_azure_marketplace_products](#test-323) +- [Test 324: browse_azure_marketplace_products](#test-324) +- [Test 325: get_azure_capacity](#test-325) +- [Test 326: get_azure_capacity](#test-326) +- [Test 327: get_azure_messaging_service_details](#test-327) +- [Test 328: get_azure_messaging_service_details](#test-328) +- [Test 329: get_azure_messaging_service_details](#test-329) +- [Test 330: get_azure_messaging_service_details](#test-330) +- [Test 331: get_azure_messaging_service_details](#test-331) +- [Test 332: get_azure_messaging_service_details](#test-332) +- [Test 333: get_azure_messaging_service_details](#test-333) +- [Test 334: get_azure_messaging_service_details](#test-334) +- [Test 335: get_azure_messaging_service_details](#test-335) +- [Test 336: get_azure_messaging_service_details](#test-336) +- [Test 337: get_azure_messaging_service_details](#test-337) +- [Test 338: get_azure_messaging_service_details](#test-338) +- [Test 339: get_azure_messaging_service_details](#test-339) +- [Test 340: get_azure_messaging_service_details](#test-340) +- [Test 341: get_azure_messaging_service_details](#test-341) +- [Test 342: get_azure_messaging_service_details](#test-342) +- [Test 343: get_azure_messaging_service_details](#test-343) +- [Test 344: get_azure_messaging_service_details](#test-344) +- [Test 345: get_azure_messaging_service_details](#test-345) +- [Test 346: get_azure_messaging_service_details](#test-346) +- [Test 347: edit_azure_data_analytics_resources](#test-347) +- [Test 348: edit_azure_data_analytics_resources](#test-348) +- [Test 349: edit_azure_data_analytics_resources](#test-349) +- [Test 350: edit_azure_data_analytics_resources](#test-350) +- [Test 351: edit_azure_data_analytics_resources](#test-351) +- [Test 352: edit_azure_data_analytics_resources](#test-352) +- [Test 353: edit_azure_data_analytics_resources](#test-353) +- [Test 354: edit_azure_data_analytics_resources](#test-354) +- [Test 355: edit_azure_data_analytics_resources](#test-355) +- [Test 356: publish_azure_eventgrid_events](#test-356) +- [Test 357: publish_azure_eventgrid_events](#test-357) +- [Test 358: publish_azure_eventgrid_events](#test-358) +- [Test 359: get_azure_data_explorer_kusto_details](#test-359) +- [Test 360: get_azure_data_explorer_kusto_details](#test-360) +- [Test 361: get_azure_data_explorer_kusto_details](#test-361) +- [Test 362: get_azure_data_explorer_kusto_details](#test-362) +- [Test 363: get_azure_data_explorer_kusto_details](#test-363) +- [Test 364: get_azure_data_explorer_kusto_details](#test-364) +- [Test 365: get_azure_data_explorer_kusto_details](#test-365) +- [Test 366: get_azure_data_explorer_kusto_details](#test-366) +- [Test 367: get_azure_data_explorer_kusto_details](#test-367) +- [Test 368: get_azure_data_explorer_kusto_details](#test-368) +- [Test 369: get_azure_data_explorer_kusto_details](#test-369) +- [Test 370: create_azure_database_admin_configurations](#test-370) +- [Test 371: create_azure_database_admin_configurations](#test-371) +- [Test 372: create_azure_database_admin_configurations](#test-372) +- [Test 373: delete_azure_database_admin_configurations](#test-373) +- [Test 374: delete_azure_database_admin_configurations](#test-374) +- [Test 375: delete_azure_database_admin_configurations](#test-375) +- [Test 376: get_azure_database_admin_configuration_details](#test-376) +- [Test 377: get_azure_database_admin_configuration_details](#test-377) +- [Test 378: get_azure_database_admin_configuration_details](#test-378) +- [Test 379: get_azure_database_admin_configuration_details](#test-379) +- [Test 380: get_azure_database_admin_configuration_details](#test-380) +- [Test 381: get_azure_database_admin_configuration_details](#test-381) +- [Test 382: get_azure_database_admin_configuration_details](#test-382) +- [Test 383: get_azure_database_admin_configuration_details](#test-383) +- [Test 384: get_azure_database_admin_configuration_details](#test-384) +- [Test 385: get_azure_container_details](#test-385) +- [Test 386: get_azure_container_details](#test-386) +- [Test 387: get_azure_container_details](#test-387) +- [Test 388: get_azure_container_details](#test-388) +- [Test 389: get_azure_container_details](#test-389) +- [Test 390: get_azure_container_details](#test-390) +- [Test 391: get_azure_container_details](#test-391) +- [Test 392: get_azure_container_details](#test-392) +- [Test 393: get_azure_container_details](#test-393) +- [Test 394: get_azure_container_details](#test-394) +- [Test 395: get_azure_container_details](#test-395) +- [Test 396: get_azure_container_details](#test-396) +- [Test 397: get_azure_container_details](#test-397) +- [Test 398: get_azure_container_details](#test-398) +- [Test 399: get_azure_container_details](#test-399) +- [Test 400: get_azure_container_details](#test-400) +- [Test 401: get_azure_container_details](#test-401) +- [Test 402: get_azure_container_details](#test-402) +- [Test 403: get_azure_container_details](#test-403) +- [Test 404: get_azure_container_details](#test-404) +- [Test 405: get_azure_container_details](#test-405) +- [Test 406: get_azure_container_details](#test-406) +- [Test 407: get_azure_virtual_desktop_details](#test-407) +- [Test 408: get_azure_virtual_desktop_details](#test-408) +- [Test 409: get_azure_virtual_desktop_details](#test-409) +- [Test 410: get_azure_signalr_details](#test-410) +- [Test 411: get_azure_signalr_details](#test-411) +- [Test 412: get_azure_signalr_details](#test-412) +- [Test 413: get_azure_signalr_details](#test-413) +- [Test 414: get_azure_signalr_details](#test-414) +- [Test 415: get_azure_signalr_details](#test-415) +- [Test 416: get_azure_confidential_ledger_entries](#test-416) +- [Test 417: get_azure_confidential_ledger_entries](#test-417) +- [Test 418: append_azure_confidential_ledger_entries](#test-418) +- [Test 419: append_azure_confidential_ledger_entries](#test-419) +- [Test 420: append_azure_confidential_ledger_entries](#test-420) +- [Test 421: append_azure_confidential_ledger_entries](#test-421) +- [Test 422: append_azure_confidential_ledger_entries](#test-422) +- [Test 423: send_azure_communication_messages](#test-423) +- [Test 424: send_azure_communication_messages](#test-424) +- [Test 425: send_azure_communication_messages](#test-425) +- [Test 426: send_azure_communication_messages](#test-426) +- [Test 427: send_azure_communication_messages](#test-427) +- [Test 428: send_azure_communication_messages](#test-428) +- [Test 429: send_azure_communication_messages](#test-429) +- [Test 430: send_azure_communication_messages](#test-430) +- [Test 431: send_azure_communication_messages](#test-431) +- [Test 432: send_azure_communication_messages](#test-432) +- [Test 433: send_azure_communication_messages](#test-433) +- [Test 434: send_azure_communication_messages](#test-434) +- [Test 435: send_azure_communication_messages](#test-435) +- [Test 436: send_azure_communication_messages](#test-436) +- [Test 437: send_azure_communication_messages](#test-437) +- [Test 438: send_azure_communication_messages](#test-438) +- [Test 439: recognize_speech_from_audio](#test-439) +- [Test 440: recognize_speech_from_audio](#test-440) +- [Test 441: recognize_speech_from_audio](#test-441) +- [Test 442: recognize_speech_from_audio](#test-442) +- [Test 443: recognize_speech_from_audio](#test-443) +- [Test 444: recognize_speech_from_audio](#test-444) +- [Test 445: recognize_speech_from_audio](#test-445) +- [Test 446: recognize_speech_from_audio](#test-446) +- [Test 447: recognize_speech_from_audio](#test-447) +- [Test 448: recognize_speech_from_audio](#test-448) --- @@ -350,9 +475,9 @@ |------|-------|------|--------| | 1 | 0.638889 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.420089 | `get_azure_security_configurations` | ❌ | -| 3 | 0.384567 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.382376 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.368405 | `get_azure_storage_details` | ❌ | +| 3 | 0.401270 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.384567 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.366619 | `get_azure_virtual_desktop_details` | ❌ | --- @@ -366,10 +491,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.415793 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.383878 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.328704 | `get_azure_security_configurations` | ❌ | -| 4 | 0.317407 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.290558 | `get_azure_storage_details` | ❌ | +| 2 | 0.328705 | `get_azure_security_configurations` | ❌ | +| 3 | 0.317407 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.312982 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.285674 | `get_azure_storage_details` | ❌ | --- @@ -383,9 +508,9 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.549609 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.418811 | `get_azure_security_configurations` | ❌ | -| 3 | 0.364712 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.363601 | `get_azure_storage_details` | ❌ | +| 2 | 0.418812 | `get_azure_security_configurations` | ❌ | +| 3 | 0.370016 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.364712 | `get_azure_load_testing_details` | ❌ | | 5 | 0.358284 | `get_azure_virtual_desktop_details` | ❌ | --- @@ -399,11 +524,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.347561 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.305506 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.278264 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.242552 | `get_azure_security_configurations` | ❌ | -| 5 | 0.206525 | `get_azure_key_vault` | ❌ | +| 1 | 0.347487 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | +| 2 | 0.278204 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.242561 | `get_azure_security_configurations` | ❌ | +| 4 | 0.238716 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.189112 | `get_azure_container_details` | ❌ | --- @@ -416,9 +541,9 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.671044 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | +| 1 | 0.671045 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.437174 | `get_azure_security_configurations` | ❌ | -| 3 | 0.399337 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.422991 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.381286 | `get_azure_load_testing_details` | ❌ | | 5 | 0.379037 | `get_azure_virtual_desktop_details` | ❌ | @@ -434,10 +559,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.322378 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.291113 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.246133 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.199535 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.193922 | `get_azure_capacity` | ❌ | +| 2 | 0.246134 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.233990 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.210046 | `get_azure_signalr_details` | ❌ | +| 5 | 0.201196 | `get_azure_resource_and_app_health_status` | ❌ | --- @@ -451,367 +576,367 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.380158 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.361939 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.303601 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.274746 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.227908 | `get_azure_container_details` | ❌ | -| 5 | 0.227901 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.235632 | `get_azure_resource_and_app_health_status` | ❌ | +| 5 | 0.227908 | `get_azure_container_details` | ❌ | --- ## Test 8 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add a CosmosDB database to app service +**Prompt:** Describe the function app in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.475947 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.460199 | `get_azure_databases_details` | ❌ | -| 3 | 0.441557 | `edit_azure_databases` | ❌ | -| 4 | 0.360392 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.353615 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.567577 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.464896 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.438587 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.414459 | `get_azure_resource_and_app_health_status` | ❌ | +| 5 | 0.391945 | `generate_azure_cli_commands` | ❌ | --- ## Test 9 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add a database connection to my app service in resource group +**Prompt:** Get configuration for function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.416690 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.399204 | `edit_azure_databases` | ❌ | -| 3 | 0.339497 | `get_azure_databases_details` | ❌ | -| 4 | 0.330529 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.325063 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.622593 | `get_azure_app_config_settings` | ❌ | +| 2 | 0.565172 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 3 | 0.480175 | `lock_unlock_azure_app_config_settings` | ❌ | +| 4 | 0.434907 | `get_azure_best_practices` | ❌ | +| 5 | 0.415185 | `edit_azure_app_config_settings` | ❌ | --- ## Test 10 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add a MySQL database to app service +**Prompt:** Get function app status for ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510959 | `edit_azure_databases` | ❌ | -| 2 | 0.461314 | `get_azure_databases_details` | ❌ | -| 3 | 0.447485 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 4 | 0.328092 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.319773 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.551165 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.444696 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.412107 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.345823 | `get_azure_signalr_details` | ❌ | +| 5 | 0.345466 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 11 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add a PostgreSQL database to app service +**Prompt:** Get information about my function app in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.488067 | `edit_azure_databases` | ❌ | -| 2 | 0.423578 | `get_azure_databases_details` | ❌ | -| 3 | 0.415055 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 4 | 0.298856 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.282619 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.606747 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.516830 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.498783 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.468835 | `get_azure_signalr_details` | ❌ | +| 5 | 0.428887 | `get_azure_messaging_service_details` | ❌ | --- ## Test 12 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add database on server to app service +**Prompt:** List all function apps in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.445528 | `edit_azure_databases` | ❌ | -| 2 | 0.440446 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 3 | 0.436024 | `get_azure_databases_details` | ❌ | -| 4 | 0.360971 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.305096 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.558485 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.427410 | `get_azure_security_configurations` | ❌ | +| 3 | 0.421965 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.420496 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.409842 | `get_azure_resource_and_app_health_status` | ❌ | --- ## Test 13 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Add database with retry policy to app service +**Prompt:** Retrieve host name and status of function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.409196 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.380948 | `edit_azure_databases` | ❌ | -| 3 | 0.359762 | `get_azure_databases_details` | ❌ | -| 4 | 0.330075 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.318303 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.545132 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.462941 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.432003 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.396163 | `get_azure_signalr_details` | ❌ | +| 5 | 0.383867 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 14 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Configure a SQL Server database for app service +**Prompt:** Show function app details for in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512464 | `edit_azure_databases` | ❌ | -| 2 | 0.497479 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 3 | 0.456119 | `lock_unlock_azure_app_config_settings` | ❌ | -| 4 | 0.438142 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.429642 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.630201 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.514721 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.434995 | `get_azure_signalr_details` | ❌ | +| 4 | 0.430445 | `deploy_azure_resources_and_applications` | ❌ | +| 5 | 0.417163 | `get_azure_messaging_service_details` | ❌ | --- ## Test 15 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Configure tenant for database in app service +**Prompt:** Show me my Azure function apps ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.438242 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.427201 | `edit_azure_databases` | ❌ | -| 3 | 0.397353 | `lock_unlock_azure_app_config_settings` | ❌ | -| 4 | 0.364309 | `get_azure_databases_details` | ❌ | -| 5 | 0.357462 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.560507 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.469871 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.462610 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.444987 | `get_azure_security_configurations` | ❌ | +| 5 | 0.437162 | `get_azure_app_config_settings` | ❌ | --- ## Test 16 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Describe the function app in resource group +**Prompt:** Show me the details for the function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.535574 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.464895 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.438587 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.412412 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.376779 | `get_azure_best_practices` | ❌ | +| 1 | 0.650735 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.570557 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.467885 | `get_azure_signalr_details` | ❌ | +| 4 | 0.426620 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.394452 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 17 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Get configuration for function app +**Prompt:** Show plan and region for function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622593 | `get_azure_app_config_settings` | ❌ | -| 2 | 0.517288 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 3 | 0.480175 | `lock_unlock_azure_app_config_settings` | ❌ | -| 4 | 0.434907 | `get_azure_best_practices` | ❌ | -| 5 | 0.415185 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.534810 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.433102 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.428962 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.390991 | `get_azure_capacity` | ❌ | +| 5 | 0.390919 | `get_azure_best_practices` | ❌ | --- ## Test 18 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Get function app status for +**Prompt:** What function apps do I have? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.488083 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.444736 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.419295 | `get_azure_resource_and_app_health_status` | ❌ | -| 4 | 0.360835 | `get_azure_storage_details` | ❌ | -| 5 | 0.345517 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.416096 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.320318 | `get_azure_resource_and_app_health_status` | ❌ | +| 3 | 0.305243 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.269663 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.263333 | `execute_azure_deployments` | ❌ | --- ## Test 19 **Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Get information about my function app in +**Prompt:** What is the status of function app ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.528620 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.516861 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.498775 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.433480 | `get_azure_storage_details` | ❌ | -| 5 | 0.427611 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.547155 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.435715 | `get_azure_resource_and_app_health_status` | ❌ | +| 3 | 0.419457 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.388241 | `deploy_azure_resources_and_applications` | ❌ | +| 5 | 0.385296 | `get_azure_signalr_details` | ❌ | --- ## Test 20 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** List all function apps in my subscription +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add a CosmosDB database to app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508222 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.448105 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.427352 | `get_azure_security_configurations` | ❌ | -| 4 | 0.421921 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.409234 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.645561 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.460200 | `get_azure_databases_details` | ❌ | +| 3 | 0.441681 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.441557 | `edit_azure_databases` | ❌ | +| 5 | 0.429157 | `create_azure_sql_databases_and_servers` | ❌ | --- ## Test 21 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Retrieve host name and status of function app +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add a database connection to my app service in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.494647 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.462941 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.441714 | `get_azure_resource_and_app_health_status` | ❌ | -| 4 | 0.385066 | `execute_azure_cli` | ❌ | -| 5 | 0.383867 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.604845 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.399204 | `edit_azure_databases` | ❌ | +| 3 | 0.374759 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.371454 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.339497 | `get_azure_databases_details` | ❌ | --- ## Test 22 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Set connection string for database in app service +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add a MySQL database to app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.468293 | `edit_azure_databases` | ❌ | -| 2 | 0.420307 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 3 | 0.395735 | `lock_unlock_azure_app_config_settings` | ❌ | -| 4 | 0.366333 | `get_azure_databases_details` | ❌ | -| 5 | 0.364732 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.662973 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.510959 | `edit_azure_databases` | ❌ | +| 3 | 0.461314 | `get_azure_databases_details` | ❌ | +| 4 | 0.409706 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.404462 | `create_azure_sql_databases_and_servers` | ❌ | --- ## Test 23 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Show function app details for in +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add a PostgreSQL database to app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.581479 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.514721 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.430445 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.411448 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.401384 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.607043 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.488067 | `edit_azure_databases` | ❌ | +| 3 | 0.423578 | `get_azure_databases_details` | ❌ | +| 4 | 0.352923 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.345103 | `rename_azure_sql_databases` | ❌ | --- ## Test 24 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Show me my Azure function apps +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add database on server to app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520882 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.469871 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.462611 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.444986 | `get_azure_security_configurations` | ❌ | -| 5 | 0.437162 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.612125 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.488271 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.455473 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.445528 | `edit_azure_databases` | ❌ | +| 5 | 0.436024 | `get_azure_databases_details` | ❌ | --- ## Test 25 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Show me the details for the function app +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Add database with retry policy to app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595243 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.570557 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.445049 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.394452 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.391607 | `get_azure_storage_details` | ❌ | +| 1 | 0.550393 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.418395 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.398384 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.380948 | `edit_azure_databases` | ❌ | +| 5 | 0.359762 | `get_azure_databases_details` | ❌ | --- ## Test 26 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** Show plan and region for function app +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Configure a SQL Server database for app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.499709 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.433102 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.428962 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.390991 | `get_azure_capacity` | ❌ | -| 5 | 0.390919 | `get_azure_best_practices` | ❌ | +| 1 | 0.637303 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.518406 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.512464 | `edit_azure_databases` | ❌ | +| 4 | 0.476589 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.468705 | `edit_azure_sql_databases_and_servers` | ❌ | --- ## Test 27 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** What function apps do I have? +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Configure tenant for database in app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.404196 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.315885 | `get_azure_resource_and_app_health_status` | ❌ | -| 3 | 0.305243 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.269663 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.262952 | `execute_azure_cli` | ❌ | +| 1 | 0.539505 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.446249 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.427201 | `edit_azure_databases` | ❌ | +| 4 | 0.397353 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.395684 | `create_azure_sql_databases_and_servers` | ❌ | --- ## Test 28 -**Expected Tool:** `get_azure_app_resource_details` -**Prompt:** What is the status of function app ? +**Expected Tool:** `add_azure_app_service_database` +**Prompt:** Set connection string for database in app service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517747 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.438212 | `get_azure_resource_and_app_health_status` | ❌ | -| 3 | 0.419467 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.388259 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.360351 | `get_azure_best_practices` | ❌ | +| 1 | 0.543995 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.468293 | `edit_azure_databases` | ❌ | +| 3 | 0.428196 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.395735 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.366333 | `get_azure_databases_details` | ❌ | --- @@ -827,8 +952,8 @@ | 1 | 0.394526 | `get_azure_database_admin_configuration_details` | ❌ | | 2 | 0.378321 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.346100 | `edit_azure_databases` | ❌ | -| 4 | 0.312577 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.311368 | `delete_azure_database_admin_configurations` | ❌ | +| 4 | 0.321543 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.313666 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -841,11 +966,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516062 | `get_azure_database_admin_configuration_details` | ❌ | -| 2 | 0.515479 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.369218 | `edit_azure_databases` | ❌ | -| 4 | 0.347208 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 5 | 0.341826 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.516088 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.515522 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.369146 | `edit_azure_databases` | ❌ | +| 4 | 0.349769 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.348912 | `create_azure_sql_databases_and_servers` | ❌ | --- @@ -860,9 +985,9 @@ |------|-------|------|--------| | 1 | 0.478446 | `get_azure_database_admin_configuration_details` | ❌ | | 2 | 0.449240 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.375171 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 4 | 0.373528 | `edit_azure_databases` | ❌ | -| 5 | 0.339970 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.378667 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.375171 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 5 | 0.373528 | `edit_azure_databases` | ❌ | --- @@ -876,10 +1001,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.437870 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.423838 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 3 | 0.422601 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.420824 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.417101 | `delete_azure_database_admin_configurations` | ❌ | +| 2 | 0.435676 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.424056 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.423838 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 5 | 0.422601 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -894,9 +1019,9 @@ |------|-------|------|--------| | 1 | 0.483707 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.470240 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.466822 | `get_azure_storage_details` | ❌ | -| 4 | 0.449054 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.439142 | `get_azure_security_configurations` | ❌ | +| 3 | 0.450588 | `get_azure_storage_details` | ❌ | +| 4 | 0.439142 | `get_azure_security_configurations` | ❌ | +| 5 | 0.419961 | `get_azure_messaging_service_details` | ❌ | --- @@ -909,11 +1034,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550049 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.447529 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.420211 | `delete_azure_database_admin_configurations` | ❌ | -| 4 | 0.415459 | `edit_azure_databases` | ❌ | -| 5 | 0.403496 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.550071 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.538282 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.450003 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.447551 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.446845 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -928,9 +1053,9 @@ |------|-------|------|--------| | 1 | 0.459813 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.353887 | `edit_azure_databases` | ❌ | -| 3 | 0.255202 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.238100 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.224949 | `get_azure_security_configurations` | ❌ | +| 3 | 0.334742 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.272054 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.255201 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -943,11 +1068,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.496481 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.418841 | `edit_azure_databases` | ❌ | -| 3 | 0.335264 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.322067 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.305614 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.496395 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.418769 | `edit_azure_databases` | ❌ | +| 3 | 0.335256 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.322093 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.316396 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -962,9 +1087,9 @@ |------|-------|------|--------| | 1 | 0.411217 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.321958 | `edit_azure_databases` | ❌ | -| 3 | 0.243270 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.230828 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.203815 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.293602 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.243270 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.234938 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -980,8 +1105,8 @@ | 1 | 0.455154 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.397754 | `edit_azure_databases` | ❌ | | 3 | 0.341460 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.323444 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.319877 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.319877 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.298417 | `create_azure_sql_databases_and_servers` | ❌ | --- @@ -996,9 +1121,9 @@ |------|-------|------|--------| | 1 | 0.416288 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.319283 | `edit_azure_databases` | ❌ | -| 3 | 0.224069 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.224061 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.219312 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.284157 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.253452 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.224069 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1013,9 +1138,9 @@ |------|-------|------|--------| | 1 | 0.375280 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.295866 | `edit_azure_databases` | ❌ | -| 3 | 0.217704 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.204516 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.202386 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.247094 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.217705 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.215628 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1029,10 +1154,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.485364 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.427326 | `get_azure_storage_details` | ❌ | -| 3 | 0.423698 | `create_azure_storage` | ❌ | -| 4 | 0.393277 | `get_azure_container_details` | ❌ | -| 5 | 0.360627 | `get_azure_cache_for_redis_details` | ❌ | +| 2 | 0.410568 | `get_azure_storage_details` | ❌ | +| 3 | 0.393277 | `get_azure_container_details` | ❌ | +| 4 | 0.379221 | `create_azure_storage` | ❌ | +| 5 | 0.360666 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -1045,11 +1170,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521537 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.388467 | `get_azure_storage_details` | ❌ | -| 3 | 0.372579 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.356233 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.340541 | `get_azure_security_configurations` | ❌ | +| 1 | 0.521563 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.403940 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.372599 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.356285 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.353188 | `get_azure_storage_details` | ❌ | --- @@ -1062,11 +1187,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.400787 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.313451 | `edit_azure_databases` | ❌ | -| 3 | 0.260782 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.239619 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.224217 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.400716 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.313467 | `edit_azure_databases` | ❌ | +| 3 | 0.301000 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.252886 | `get_azure_ai_resources_details` | ❌ | +| 5 | 0.246393 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1079,11 +1204,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.357470 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.282259 | `edit_azure_databases` | ❌ | -| 3 | 0.237418 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.230730 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.211605 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.357640 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.282367 | `edit_azure_databases` | ❌ | +| 3 | 0.276725 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.230752 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.228384 | `get_azure_ai_resources_details` | ❌ | --- @@ -1097,10 +1222,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.558434 | `get_azure_database_admin_configuration_details` | ❌ | -| 2 | 0.530328 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.530329 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.486536 | `get_azure_app_config_settings` | ❌ | | 4 | 0.468156 | `edit_azure_databases` | ❌ | -| 5 | 0.421194 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.463670 | `create_azure_sql_databases_and_servers` | ❌ | --- @@ -1116,8 +1241,8 @@ | 1 | 0.459302 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 2 | 0.424009 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.422684 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 4 | 0.399212 | `get_azure_storage_details` | ❌ | -| 5 | 0.389735 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.409588 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.386413 | `get_azure_capacity` | ❌ | --- @@ -1133,8 +1258,8 @@ | 1 | 0.318903 | `edit_azure_databases` | ❌ | | 2 | 0.251389 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.215865 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.165104 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.164953 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.168314 | `get_azure_signalr_details` | ❌ | +| 5 | 0.165104 | `create_azure_database_admin_configurations` | ❌ | --- @@ -1148,10 +1273,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.494153 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.446933 | `get_azure_storage_details` | ❌ | -| 3 | 0.421776 | `get_azure_security_configurations` | ❌ | -| 4 | 0.396421 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.389268 | `browse_azure_marketplace_products` | ❌ | +| 2 | 0.423273 | `get_azure_storage_details` | ❌ | +| 3 | 0.421777 | `get_azure_security_configurations` | ❌ | +| 4 | 0.389268 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.388560 | `get_azure_container_details` | ❌ | --- @@ -1166,9 +1291,9 @@ |------|-------|------|--------| | 1 | 0.404968 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.357185 | `edit_azure_databases` | ❌ | -| 3 | 0.270829 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.234680 | `delete_azure_database_admin_configurations` | ❌ | -| 5 | 0.227130 | `get_azure_security_configurations` | ❌ | +| 3 | 0.274336 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.270829 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.264802 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1184,8 +1309,8 @@ | 1 | 0.380591 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.354214 | `edit_azure_databases` | ❌ | | 3 | 0.274988 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.240477 | `delete_azure_database_admin_configurations` | ❌ | -| 5 | 0.234628 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.253387 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.240477 | `delete_azure_database_admin_configurations` | ❌ | --- @@ -1202,7 +1327,7 @@ | 2 | 0.345881 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.319072 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.282061 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.217174 | `delete_azure_database_admin_configurations` | ❌ | +| 5 | 0.252978 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1219,7 +1344,7 @@ | 2 | 0.305067 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.302014 | `get_azure_databases_details` | ✅ **EXPECTED** | | 4 | 0.250062 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.200991 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.214600 | `update_azure_managed_lustre_filesystems` | ❌ | --- @@ -1233,10 +1358,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.467713 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.403564 | `create_azure_storage` | ❌ | -| 3 | 0.391129 | `get_azure_storage_details` | ❌ | -| 4 | 0.377338 | `get_azure_container_details` | ❌ | -| 5 | 0.327793 | `get_azure_cache_for_redis_details` | ❌ | +| 2 | 0.380264 | `get_azure_storage_details` | ❌ | +| 3 | 0.377338 | `get_azure_container_details` | ❌ | +| 4 | 0.351073 | `create_azure_storage` | ❌ | +| 5 | 0.327822 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -1251,9 +1376,9 @@ |------|-------|------|--------| | 1 | 0.488113 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 2 | 0.481986 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 3 | 0.473445 | `get_azure_storage_details` | ❌ | -| 4 | 0.458261 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.444953 | `get_azure_security_configurations` | ❌ | +| 3 | 0.458807 | `get_azure_storage_details` | ❌ | +| 4 | 0.444953 | `get_azure_security_configurations` | ❌ | +| 5 | 0.430545 | `get_azure_messaging_service_details` | ❌ | --- @@ -1266,11 +1391,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.496822 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.373904 | `get_azure_storage_details` | ❌ | +| 1 | 0.496823 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.380341 | `rename_azure_sql_databases` | ❌ | | 3 | 0.368375 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.336256 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.323139 | `get_azure_security_configurations` | ❌ | +| 4 | 0.340381 | `get_azure_storage_details` | ❌ | +| 5 | 0.336644 | `add_azure_app_service_database` | ❌ | --- @@ -1283,10 +1408,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549663 | `get_azure_database_admin_configuration_details` | ❌ | -| 2 | 0.489035 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.483888 | `get_azure_storage_details` | ❌ | -| 4 | 0.476130 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.549664 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.500276 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.493713 | `get_azure_signalr_details` | ❌ | +| 4 | 0.489035 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 5 | 0.464348 | `get_azure_app_config_settings` | ❌ | --- @@ -1302,9 +1427,9 @@ |------|-------|------|--------| | 1 | 0.430729 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.415590 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.358991 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.354816 | `edit_azure_databases` | ❌ | -| 5 | 0.343500 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.354815 | `edit_azure_databases` | ❌ | +| 4 | 0.343500 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.336660 | `rename_azure_sql_databases` | ❌ | --- @@ -1318,10 +1443,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.425531 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.382341 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.377916 | `get_azure_storage_details` | ❌ | -| 4 | 0.346397 | `get_azure_container_details` | ❌ | -| 5 | 0.328296 | `browse_azure_marketplace_products` | ❌ | +| 2 | 0.418012 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.380456 | `get_azure_ai_resources_details` | ❌ | +| 4 | 0.349347 | `get_azure_storage_details` | ❌ | +| 5 | 0.346397 | `get_azure_container_details` | ❌ | --- @@ -1336,9 +1461,9 @@ |------|-------|------|--------| | 1 | 0.443295 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.358413 | `edit_azure_databases` | ❌ | -| 3 | 0.250030 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.239673 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.215038 | `get_azure_cache_for_redis_details` | ❌ | +| 3 | 0.322639 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.287308 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.277441 | `create_azure_sql_databases_and_servers` | ❌ | --- @@ -1351,11 +1476,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.511744 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.461478 | `edit_azure_databases` | ❌ | -| 3 | 0.364350 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.353379 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.349028 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.511745 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.461477 | `edit_azure_databases` | ❌ | +| 3 | 0.365358 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.364350 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.353379 | `browse_azure_marketplace_products` | ❌ | --- @@ -1370,9 +1495,9 @@ |------|-------|------|--------| | 1 | 0.406462 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.332482 | `edit_azure_databases` | ❌ | -| 3 | 0.245580 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.227159 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.203381 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.277752 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.247838 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.245580 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1385,11 +1510,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.469147 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.433769 | `edit_azure_databases` | ❌ | -| 3 | 0.360519 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.343634 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.339215 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.469057 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.433648 | `edit_azure_databases` | ❌ | +| 3 | 0.360426 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.343503 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.339133 | `browse_azure_marketplace_products` | ❌ | --- @@ -1402,11 +1527,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.383189 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 1 | 0.382863 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.314466 | `edit_azure_databases` | ❌ | -| 3 | 0.225609 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.221419 | `get_azure_best_practices` | ❌ | -| 5 | 0.219501 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.225336 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.224832 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.221369 | `get_azure_best_practices` | ❌ | --- @@ -1419,11 +1544,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.343871 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.284665 | `edit_azure_databases` | ❌ | -| 3 | 0.214231 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.199822 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.193724 | `get_azure_best_practices` | ❌ | +| 1 | 0.343844 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.284626 | `edit_azure_databases` | ❌ | +| 3 | 0.214230 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.201567 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.199864 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1438,9 +1563,9 @@ |------|-------|------|--------| | 1 | 0.421279 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.335190 | `edit_azure_databases` | ❌ | -| 3 | 0.249102 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.243773 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.210444 | `get_azure_security_configurations` | ❌ | +| 3 | 0.262481 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.256053 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.249102 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1453,11 +1578,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.375595 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.310370 | `edit_azure_databases` | ❌ | -| 3 | 0.230608 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.224141 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.188657 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.375836 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.311404 | `edit_azure_databases` | ❌ | +| 3 | 0.230790 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.226153 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.224314 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1480,4381 +1605,6506 @@ ## Test 68 -**Expected Tool:** `edit_azure_databases` -**Prompt:** Enable replication for my PostgreSQL server +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Create a new Azure SQL server named in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.340843 | `edit_azure_databases` | ✅ **EXPECTED** | -| 2 | 0.250565 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.234603 | `get_azure_databases_details` | ❌ | -| 4 | 0.212755 | `delete_azure_database_admin_configurations` | ❌ | -| 5 | 0.160632 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.544634 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.496642 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.474577 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.398467 | `delete_azure_database_admin_configurations` | ❌ | +| 5 | 0.395746 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 69 -**Expected Tool:** `edit_azure_databases` -**Prompt:** Set connection timeout to 20 seconds for my MySQL server +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Create a new database called on SQL server in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.380868 | `edit_azure_databases` | ✅ **EXPECTED** | -| 2 | 0.269323 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.251993 | `delete_azure_database_admin_configurations` | ❌ | -| 4 | 0.213409 | `get_azure_databases_details` | ❌ | -| 5 | 0.174321 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.520385 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.513431 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.384182 | `add_azure_app_service_database` | ❌ | +| 4 | 0.381023 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.376654 | `create_azure_database_admin_configurations` | ❌ | --- ## Test 70 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Analyze the performance trends and response times for Application Insights resource over the last +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Create a new SQL database named in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521504 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.402104 | `create_azure_load_testing` | ❌ | -| 3 | 0.398005 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.397775 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.383437 | `get_azure_capacity` | ❌ | +| 1 | 0.492741 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.448210 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.371279 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.349862 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.339719 | `edit_azure_databases` | ❌ | --- ## Test 71 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Check the availability metrics for my Application Insights resource for the last +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Create a SQL database with Basic tier in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542512 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.426884 | `get_azure_capacity` | ❌ | -| 3 | 0.381282 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.370434 | `get_azure_ai_resources_details` | ❌ | -| 5 | 0.368141 | `create_azure_load_testing` | ❌ | +| 1 | 0.586664 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.448354 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.429547 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.414021 | `edit_azure_databases` | ❌ | +| 5 | 0.390212 | `add_azure_app_service_database` | ❌ | --- ## Test 72 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Get metric definitions for from the namespace +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Create an Azure SQL server with name in location with admin user ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.339028 | `get_azure_storage_details` | ❌ | -| 2 | 0.315432 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 3 | 0.272938 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.271450 | `get_azure_capacity` | ❌ | -| 5 | 0.269633 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.502365 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.468619 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.424715 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.411480 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.411012 | `rename_azure_sql_databases` | ❌ | --- ## Test 73 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Get the metric for over the last with intervals +**Expected Tool:** `create_azure_sql_databases_and_servers` +**Prompt:** Set up a new SQL server called in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.337793 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.260055 | `get_azure_storage_details` | ❌ | -| 3 | 0.253643 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.243989 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.237046 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.545618 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.485689 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.442793 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.407416 | `edit_azure_databases` | ❌ | +| 5 | 0.391782 | `add_azure_app_service_database` | ❌ | --- ## Test 74 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Get the availability status for resource +**Expected Tool:** `rename_azure_sql_databases` +**Prompt:** Rename my Azure SQL database to on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.409198 | `get_azure_storage_details` | ❌ | -| 2 | 0.379831 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 3 | 0.374257 | `get_azure_capacity` | ❌ | -| 4 | 0.325422 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.305299 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.794416 | `rename_azure_sql_databases` | ✅ **EXPECTED** | +| 2 | 0.541638 | `edit_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.505153 | `edit_azure_databases` | ❌ | +| 4 | 0.475451 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.437195 | `create_azure_database_admin_configurations` | ❌ | --- ## Test 75 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Investigate error rates and failed requests for Application Insights resource for the last +**Expected Tool:** `rename_azure_sql_databases` +**Prompt:** Rename the SQL database on server to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.486374 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.389526 | `get_azure_capacity` | ❌ | -| 3 | 0.375039 | `create_azure_load_testing` | ❌ | -| 4 | 0.370705 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.369547 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.689327 | `rename_azure_sql_databases` | ✅ **EXPECTED** | +| 2 | 0.436705 | `edit_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.396117 | `edit_azure_databases` | ❌ | +| 4 | 0.358714 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.323652 | `get_azure_databases_details` | ❌ | --- ## Test 76 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List all available table types in the Log Analytics workspace +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Delete SQL server permanently ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.427197 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.407964 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.398563 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.396444 | `get_azure_databases_details` | ❌ | -| 5 | 0.383614 | `create_azure_workbooks` | ❌ | +| 1 | 0.477760 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.442463 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.353822 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.350061 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.314143 | `edit_azure_databases` | ❌ | --- ## Test 77 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List all Azure Managed Grafana in one subscription +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Delete the Azure SQL server from resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.470427 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.455463 | `get_azure_databases_details` | ❌ | -| 3 | 0.445863 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.440818 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.438518 | `get_azure_security_configurations` | ❌ | +| 1 | 0.530935 | `delete_azure_database_admin_configurations` | ❌ | +| 2 | 0.524563 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.470808 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.453765 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.417285 | `edit_azure_data_analytics_resources` | ❌ | --- ## Test 78 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List all Log Analytics workspaces in my subscription +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Delete the database called on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498165 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.463936 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.452333 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.437304 | `create_azure_workbooks` | ❌ | -| 5 | 0.419394 | `get_azure_security_configurations` | ❌ | +| 1 | 0.425106 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.402144 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.303504 | `get_azure_databases_details` | ❌ | +| 4 | 0.292939 | `edit_azure_databases` | ❌ | +| 5 | 0.285850 | `delete_azure_database_admin_configurations` | ❌ | --- ## Test 79 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List all monitored resources in the Datadog resource +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Delete the SQL database from server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.419335 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.356965 | `get_azure_load_testing_details` | ❌ | -| 3 | 0.349385 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.331350 | `get_azure_storage_details` | ❌ | -| 5 | 0.321614 | `get_azure_cache_for_redis_details` | ❌ | +| 1 | 0.480068 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 2 | 0.441425 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.354135 | `delete_azure_database_admin_configurations` | ❌ | +| 4 | 0.316716 | `edit_azure_databases` | ❌ | +| 5 | 0.312687 | `get_azure_databases_details` | ❌ | --- ## Test 80 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List all tables in the Log Analytics workspace +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Remove database from SQL server in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.428737 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.418527 | `get_azure_workbooks_details` | ❌ | -| 3 | 0.398748 | `create_azure_workbooks` | ❌ | -| 4 | 0.393663 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.376728 | `get_azure_databases_details` | ❌ | +| 1 | 0.479510 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.465178 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.420526 | `delete_azure_database_admin_configurations` | ❌ | +| 4 | 0.361867 | `edit_azure_data_analytics_resources` | ❌ | +| 5 | 0.356097 | `create_azure_sql_databases_and_servers` | ❌ | --- ## Test 81 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List availability status for all resources in my subscription +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Remove the SQL server from my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.536715 | `get_azure_storage_details` | ❌ | -| 2 | 0.525913 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.486032 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 4 | 0.478110 | `get_azure_capacity` | ❌ | -| 5 | 0.460629 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.466825 | `delete_azure_database_admin_configurations` | ❌ | +| 2 | 0.450380 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.395039 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.393448 | `edit_azure_databases` | ❌ | +| 5 | 0.370137 | `create_azure_database_admin_configurations` | ❌ | --- ## Test 82 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List code optimization recommendations across my Application Insights components +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Scale SQL database on server to use SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.476038 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.430555 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.423122 | `get_azure_best_practices` | ❌ | -| 4 | 0.369889 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.368140 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.447788 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.446405 | `edit_azure_databases` | ❌ | +| 3 | 0.444369 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 4 | 0.434542 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.348544 | `get_azure_databases_details` | ❌ | --- ## Test 83 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** List profiler recommendations for Application Insights in resource group +**Expected Tool:** `edit_azure_sql_databases_and_servers` +**Prompt:** Update the performance tier of SQL database on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.548400 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.453872 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.440214 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.416257 | `get_azure_security_configurations` | ❌ | -| 5 | 0.416176 | `get_azure_capacity` | ❌ | +| 1 | 0.595423 | `create_azure_sql_databases_and_servers` | ❌ | +| 2 | 0.550251 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.515796 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.506574 | `edit_azure_databases` | ❌ | +| 5 | 0.372785 | `get_azure_databases_details` | ❌ | --- ## Test 84 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Please help me diagnose issues with my app using app lens +**Expected Tool:** `edit_azure_databases` +**Prompt:** Enable replication for my PostgreSQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.378029 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.272313 | `get_azure_app_resource_details` | ❌ | -| 3 | 0.249202 | `execute_azure_cli` | ❌ | -| 4 | 0.247328 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.246532 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.340843 | `edit_azure_databases` | ✅ **EXPECTED** | +| 2 | 0.250565 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.234603 | `get_azure_databases_details` | ❌ | +| 4 | 0.220308 | `add_azure_app_service_database` | ❌ | +| 5 | 0.212755 | `delete_azure_database_admin_configurations` | ❌ | --- ## Test 85 -**Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Query the metric for for the last +**Expected Tool:** `edit_azure_databases` +**Prompt:** Set connection timeout to 20 seconds for my MySQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.344644 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.269942 | `get_azure_storage_details` | ❌ | -| 3 | 0.264069 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.253364 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.252579 | `get_azure_capacity` | ❌ | +| 1 | 0.380868 | `edit_azure_databases` | ✅ **EXPECTED** | +| 2 | 0.269323 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.251993 | `delete_azure_database_admin_configurations` | ❌ | +| 4 | 0.213410 | `get_azure_databases_details` | ❌ | +| 5 | 0.195757 | `rename_azure_sql_databases` | ❌ | --- ## Test 86 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me all available metrics and their definitions for storage account +**Prompt:** Analyze the performance trends and response times for Application Insights resource over the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577907 | `get_azure_storage_details` | ❌ | -| 2 | 0.430781 | `get_azure_capacity` | ❌ | -| 3 | 0.399711 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 4 | 0.392511 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.383711 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.518830 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.421973 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.402104 | `create_azure_load_testing` | ❌ | +| 4 | 0.398005 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.397774 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 87 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me code optimization recommendations for all Application Insights resources in my subscription +**Prompt:** Check the availability metrics for my Application Insights resource for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512832 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.492356 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.464604 | `get_azure_best_practices` | ❌ | -| 4 | 0.421210 | `get_azure_capacity` | ❌ | -| 5 | 0.400675 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.510833 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.472122 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.459627 | `update_azure_monitor_webtests` | ❌ | +| 4 | 0.426727 | `get_azure_capacity` | ❌ | +| 5 | 0.381483 | `get_azure_load_testing_details` | ❌ | --- ## Test 88 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me my Log Analytics workspaces +**Prompt:** Get metric definitions for from the namespace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520253 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.488696 | `create_azure_workbooks` | ❌ | -| 3 | 0.451337 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.410409 | `get_azure_security_configurations` | ❌ | -| 5 | 0.405448 | `edit_azure_workbooks` | ❌ | +| 1 | 0.345032 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.302173 | `get_azure_storage_details` | ❌ | +| 3 | 0.285925 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 4 | 0.272938 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.271450 | `get_azure_capacity` | ❌ | --- ## Test 89 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me performance improvement recommendations from Application Insights +**Prompt:** Get the metric for over the last with intervals ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.485261 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.431076 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.392623 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.368812 | `get_azure_capacity` | ❌ | -| 5 | 0.356368 | `get_azure_best_practices` | ❌ | +| 1 | 0.314111 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.257445 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.253789 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.244128 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.239832 | `get_azure_signalr_details` | ❌ | --- ## Test 90 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the available table types in the Log Analytics workspace +**Prompt:** Get the availability status for resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.438406 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.405799 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.396006 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.387063 | `create_azure_workbooks` | ❌ | -| 5 | 0.384415 | `get_azure_databases_details` | ❌ | +| 1 | 0.374257 | `get_azure_capacity` | ❌ | +| 2 | 0.338439 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 3 | 0.336667 | `get_azure_storage_details` | ❌ | +| 4 | 0.325422 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.305299 | `get_azure_virtual_desktop_details` | ❌ | --- ## Test 91 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the health status of all my Azure resources +**Prompt:** Investigate error rates and failed requests for Application Insights resource for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.582586 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.551904 | `get_azure_storage_details` | ❌ | -| 3 | 0.488089 | `get_azure_capacity` | ❌ | -| 4 | 0.475264 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.471465 | `get_azure_security_configurations` | ❌ | +| 1 | 0.486070 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.406567 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.389526 | `get_azure_capacity` | ❌ | +| 4 | 0.385006 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.375039 | `create_azure_load_testing` | ❌ | --- ## Test 92 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the health status of entity in the Log Analytics workspace +**Prompt:** List active service health events in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.569558 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.364021 | `get_azure_storage_details` | ❌ | -| 3 | 0.359140 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.340267 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.334046 | `create_azure_workbooks` | ❌ | +| 1 | 0.508303 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.487675 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.384016 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.380883 | `get_azure_signalr_details` | ❌ | +| 5 | 0.379399 | `publish_azure_eventgrid_events` | ❌ | --- ## Test 93 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the health status of the storage account +**Prompt:** List all available table types in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.560849 | `get_azure_storage_details` | ❌ | -| 2 | 0.406534 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 3 | 0.400710 | `create_azure_storage` | ❌ | -| 4 | 0.368175 | `get_azure_capacity` | ❌ | -| 5 | 0.339439 | `get_azure_security_configurations` | ❌ | +| 1 | 0.423129 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.407964 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.398563 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.396444 | `get_azure_databases_details` | ❌ | +| 5 | 0.383614 | `create_azure_workbooks` | ❌ | --- ## Test 94 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the Log Analytics workspaces in my subscription +**Prompt:** List all Azure Managed Grafana in one subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.523533 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.479440 | `create_azure_workbooks` | ❌ | -| 3 | 0.458853 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.456725 | `get_azure_workbooks_details` | ❌ | -| 5 | 0.418907 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.455463 | `get_azure_databases_details` | ❌ | +| 2 | 0.449685 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 3 | 0.445863 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.438518 | `get_azure_security_configurations` | ❌ | +| 5 | 0.423144 | `browse_azure_marketplace_products` | ❌ | --- ## Test 95 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the logs for the past hour for the resource in the Log Analytics workspace +**Prompt:** List all Log Analytics workspaces in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.457886 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.365358 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.333901 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.332481 | `get_azure_capacity` | ❌ | -| 5 | 0.329720 | `get_azure_workbooks_details` | ❌ | +| 1 | 0.492563 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.463935 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.452333 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.437304 | `create_azure_workbooks` | ❌ | +| 5 | 0.419394 | `get_azure_security_configurations` | ❌ | --- ## Test 96 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the logs for the past hour in the Log Analytics workspace +**Prompt:** List all monitored resources in the Datadog resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.445778 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.375709 | `create_azure_workbooks` | ❌ | -| 3 | 0.357168 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.336058 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.324855 | `edit_azure_workbooks` | ❌ | +| 1 | 0.400161 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.356965 | `get_azure_load_testing_details` | ❌ | +| 3 | 0.349385 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.321621 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.318657 | `get_azure_messaging_service_details` | ❌ | --- ## Test 97 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the monitored resources in the Datadog resource +**Prompt:** List all service health events in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.422735 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.354368 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.346010 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.321471 | `get_azure_storage_details` | ❌ | -| 5 | 0.316126 | `get_azure_cache_for_redis_details` | ❌ | +| 1 | 0.514546 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.487976 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.381633 | `get_azure_signalr_details` | ❌ | +| 4 | 0.372535 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.371822 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 98 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Show me the tables in the Log Analytics workspace +**Prompt:** List all tables in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.435717 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.420831 | `get_azure_workbooks_details` | ❌ | -| 3 | 0.400964 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.400018 | `create_azure_workbooks` | ❌ | -| 5 | 0.367191 | `get_azure_databases_details` | ❌ | +| 1 | 0.429580 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.418506 | `get_azure_workbooks_details` | ❌ | +| 3 | 0.398756 | `create_azure_workbooks` | ❌ | +| 4 | 0.393661 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.376718 | `get_azure_databases_details` | ❌ | --- ## Test 99 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** Use app lens to check why my app is slow? +**Prompt:** List availability status for all resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.357303 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.258469 | `get_azure_app_resource_details` | ❌ | -| 3 | 0.234720 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.221280 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.214378 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.525913 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.482171 | `get_azure_storage_details` | ❌ | +| 3 | 0.479549 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.478110 | `get_azure_capacity` | ❌ | +| 5 | 0.465854 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | --- ## Test 100 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** What does app lens say is wrong with my service? +**Prompt:** List code optimization recommendations across my Application Insights components ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.322056 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.275921 | `get_azure_app_resource_details` | ❌ | -| 3 | 0.212918 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.205828 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.164353 | `get_azure_databases_details` | ❌ | +| 1 | 0.474724 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.430555 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.423122 | `get_azure_best_practices` | ❌ | +| 4 | 0.407439 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.368902 | `create_azure_monitor_webtests` | ❌ | --- ## Test 101 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** What is the availability status of virtual machine in resource group ? +**Prompt:** List profiler recommendations for Application Insights in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.421696 | `get_azure_capacity` | ❌ | -| 2 | 0.410736 | `get_azure_storage_details` | ❌ | -| 3 | 0.408348 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 4 | 0.404977 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.380088 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.531640 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.453872 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.439990 | `get_azure_ai_resources_details` | ❌ | +| 4 | 0.416257 | `get_azure_security_configurations` | ❌ | +| 5 | 0.416176 | `get_azure_capacity` | ❌ | --- ## Test 102 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** What metric definitions are available for the Application Insights resource +**Prompt:** List the activity logs of the last month for ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517790 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.397417 | `get_azure_capacity` | ❌ | -| 3 | 0.382899 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.379035 | `get_azure_storage_details` | ❌ | -| 5 | 0.378286 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.340942 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.293858 | `get_azure_load_testing_details` | ❌ | +| 3 | 0.279505 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.269796 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.260043 | `get_azure_capacity` | ❌ | --- ## Test 103 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** What resources in resource group have health issues? +**Prompt:** Please help me diagnose issues with my app using app lens ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510381 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.445813 | `get_azure_storage_details` | ❌ | -| 3 | 0.423077 | `get_azure_capacity` | ❌ | -| 4 | 0.389687 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.382850 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.348670 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.247328 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.246532 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.235710 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.235128 | `get_azure_app_resource_details` | ❌ | --- ## Test 104 **Expected Tool:** `get_azure_resource_and_app_health_status` -**Prompt:** What's the request per second rate for my Application Insights resource over the last +**Prompt:** Query the metric for for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.434104 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.396015 | `get_azure_capacity` | ❌ | -| 3 | 0.369685 | `create_azure_load_testing` | ❌ | -| 4 | 0.353095 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.326466 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.330121 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.264069 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.253364 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.252579 | `get_azure_capacity` | ❌ | +| 5 | 0.245564 | `get_azure_messaging_service_details` | ❌ | --- ## Test 105 -**Expected Tool:** `deploy_resources_and_applications_to_azure` -**Prompt:** Create a plan to deploy this application to azure +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me all available metrics and their definitions for storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.640058 | `deploy_resources_and_applications_to_azure` | ✅ **EXPECTED** | -| 2 | 0.519278 | `deploy_azure_ai_models` | ❌ | -| 3 | 0.479918 | `get_azure_best_practices` | ❌ | -| 4 | 0.454755 | `execute_azure_developer_cli` | ❌ | -| 5 | 0.453039 | `design_azure_architecture` | ❌ | +| 1 | 0.558407 | `get_azure_storage_details` | ❌ | +| 2 | 0.430781 | `get_azure_capacity` | ❌ | +| 3 | 0.405616 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.403095 | `create_azure_storage` | ❌ | +| 5 | 0.390433 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | --- ## Test 106 -**Expected Tool:** `deploy_resources_and_applications_to_azure` -**Prompt:** How can I create a CI/CD pipeline to deploy this app to Azure? +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me Azure service health events for subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.578477 | `deploy_resources_and_applications_to_azure` | ✅ **EXPECTED** | -| 2 | 0.477740 | `deploy_azure_ai_models` | ❌ | -| 3 | 0.437318 | `execute_azure_developer_cli` | ❌ | -| 4 | 0.410719 | `get_azure_best_practices` | ❌ | -| 5 | 0.401777 | `execute_azure_cli` | ❌ | +| 1 | 0.496308 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.467641 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.424333 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.392392 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.387040 | `get_azure_app_resource_details` | ❌ | --- ## Test 107 -**Expected Tool:** `deploy_resources_and_applications_to_azure` -**Prompt:** Show me the log of the application deployed by azd +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me code optimization recommendations for all Application Insights resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533141 | `execute_azure_developer_cli` | ❌ | -| 2 | 0.522934 | `deploy_resources_and_applications_to_azure` | ✅ **EXPECTED** | -| 3 | 0.449771 | `get_azure_resource_and_app_health_status` | ❌ | -| 4 | 0.396308 | `execute_azure_cli` | ❌ | -| 5 | 0.393841 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.508215 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.492356 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.464604 | `get_azure_best_practices` | ❌ | +| 4 | 0.421210 | `get_azure_capacity` | ❌ | +| 5 | 0.418559 | `create_azure_monitor_webtests` | ❌ | --- ## Test 108 -**Expected Tool:** `deploy_resources_and_applications_to_azure` -**Prompt:** Show me the rules to generate bicep scripts +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me my Log Analytics workspaces ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.488008 | `get_azure_best_practices` | ❌ | -| 2 | 0.384841 | `deploy_resources_and_applications_to_azure` | ✅ **EXPECTED** | -| 3 | 0.325324 | `create_azure_database_admin_configurations` | ❌ | -| 4 | 0.315430 | `execute_azure_cli` | ❌ | -| 5 | 0.313187 | `search_microsoft_docs` | ❌ | +| 1 | 0.506190 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.488696 | `create_azure_workbooks` | ❌ | +| 3 | 0.451336 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.410409 | `get_azure_security_configurations` | ❌ | +| 5 | 0.405448 | `edit_azure_workbooks` | ❌ | --- ## Test 109 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** List all App Configuration stores in my subscription +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me performance improvement recommendations from Application Insights ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549804 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.418698 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.401422 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.400003 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.388838 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.472052 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.434889 | `update_azure_monitor_webtests` | ❌ | +| 3 | 0.411997 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.398711 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.396968 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- ## Test 110 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** List all key-value settings in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me planned maintenance events for my Azure services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.605174 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.469735 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.413315 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.310582 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.304660 | `update_azure_load_testing_configurations` | ❌ | +| 1 | 0.478758 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.467913 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.445424 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.435005 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.427813 | `get_azure_capacity` | ❌ | --- ## Test 111 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** Show me my App Configuration stores +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the available table types in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517123 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.397359 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.318242 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.302856 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.300071 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.428340 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.405799 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.396006 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.387063 | `create_azure_workbooks` | ❌ | +| 5 | 0.384415 | `get_azure_databases_details` | ❌ | --- ## Test 112 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** Show me the App Configuration stores in my subscription +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the health status of all my Azure resources ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.564754 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.445478 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.396571 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.382377 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.377431 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.575009 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.488089 | `get_azure_capacity` | ❌ | +| 3 | 0.475264 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.473441 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.471465 | `get_azure_security_configurations` | ❌ | --- ## Test 113 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** Show me the key-value settings in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the health status of entity using the health model ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619236 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.496884 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.413994 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.320934 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.303617 | `update_azure_load_testing_configurations` | ❌ | +| 1 | 0.311450 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.230296 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.209276 | `get_azure_capacity` | ❌ | +| 4 | 0.201724 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.193404 | `audit_azure_resources_compliance` | ❌ | --- ## Test 114 -**Expected Tool:** `get_azure_app_config_settings` -**Prompt:** Show the content for the key in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the health status of the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.473674 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.397489 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.319847 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.290485 | `get_azure_key_vault` | ❌ | -| 5 | 0.227918 | `get_azure_container_details` | ❌ | +| 1 | 0.494538 | `get_azure_storage_details` | ❌ | +| 2 | 0.403753 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 3 | 0.395192 | `create_azure_storage` | ❌ | +| 4 | 0.368143 | `get_azure_capacity` | ❌ | +| 5 | 0.339448 | `get_azure_security_configurations` | ❌ | --- ## Test 115 -**Expected Tool:** `edit_azure_app_config_settings` -**Prompt:** Delete the key in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the Log Analytics workspaces in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.480490 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.419225 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.386234 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.236794 | `edit_azure_workbooks` | ❌ | -| 5 | 0.226127 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.513546 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.479440 | `create_azure_workbooks` | ❌ | +| 3 | 0.458853 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.456725 | `get_azure_workbooks_details` | ❌ | +| 5 | 0.418907 | `browse_azure_marketplace_products` | ❌ | --- ## Test 116 -**Expected Tool:** `edit_azure_app_config_settings` -**Prompt:** Set the key in App Configuration store to +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the logs for the past hour for the resource in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.454677 | `lock_unlock_azure_app_config_settings` | ❌ | -| 2 | 0.419522 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.418815 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | -| 4 | 0.251838 | `update_azure_load_testing_configurations` | ❌ | -| 5 | 0.227102 | `edit_azure_databases` | ❌ | +| 1 | 0.464430 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.365333 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.333899 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.332443 | `get_azure_capacity` | ❌ | +| 5 | 0.329728 | `get_azure_workbooks_details` | ❌ | --- ## Test 117 -**Expected Tool:** `lock_unlock_azure_app_config_settings` -**Prompt:** Lock the key in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the logs for the past hour in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.523446 | `lock_unlock_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.367924 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.324652 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.206577 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.186093 | `update_azure_load_testing_configurations` | ❌ | +| 1 | 0.453704 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.375709 | `create_azure_workbooks` | ❌ | +| 3 | 0.357168 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.336058 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.324855 | `edit_azure_workbooks` | ❌ | --- ## Test 118 -**Expected Tool:** `lock_unlock_azure_app_config_settings` -**Prompt:** Unlock the key in App Configuration store +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the monitored resources in the Datadog resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552583 | `lock_unlock_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.393938 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.339108 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.240636 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.224555 | `get_azure_key_vault` | ❌ | +| 1 | 0.398557 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.354368 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.346010 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.321110 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.316100 | `get_azure_cache_for_redis_details` | ❌ | --- ## Test 119 -**Expected Tool:** `edit_azure_workbooks` -**Prompt:** Delete the workbook with resource ID +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Show me the tables in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.505878 | `edit_azure_workbooks` | ✅ **EXPECTED** | -| 2 | 0.375642 | `create_azure_workbooks` | ❌ | -| 3 | 0.362979 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.265457 | `edit_azure_app_config_settings` | ❌ | -| 5 | 0.188350 | `create_azure_load_testing` | ❌ | +| 1 | 0.433671 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.420831 | `get_azure_workbooks_details` | ❌ | +| 3 | 0.400964 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.400018 | `create_azure_workbooks` | ❌ | +| 5 | 0.367191 | `get_azure_databases_details` | ❌ | --- ## Test 120 -**Expected Tool:** `edit_azure_workbooks` -**Prompt:** Update the workbook with a new text step +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** Use app lens to check why my app is slow? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.496535 | `edit_azure_workbooks` | ✅ **EXPECTED** | -| 2 | 0.413187 | `create_azure_workbooks` | ❌ | -| 3 | 0.327796 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.236165 | `update_azure_load_testing_configurations` | ❌ | -| 5 | 0.216298 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.317206 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.234984 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.221280 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.214378 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.213578 | `get_azure_databases_details` | ❌ | --- ## Test 121 -**Expected Tool:** `create_azure_workbooks` -**Prompt:** Create a new workbook named +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What does app lens say is wrong with my service? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555073 | `create_azure_workbooks` | ✅ **EXPECTED** | -| 2 | 0.400619 | `edit_azure_workbooks` | ❌ | -| 3 | 0.371495 | `get_azure_workbooks_details` | ❌ | -| 4 | 0.196704 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.157512 | `create_azure_storage` | ❌ | +| 1 | 0.299595 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.250948 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.212919 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.205952 | `add_azure_app_service_database` | ❌ | +| 5 | 0.201566 | `get_azure_signalr_details` | ❌ | --- ## Test 122 -**Expected Tool:** `get_azure_workbooks_details` -**Prompt:** Get information about the workbook with resource ID +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What is the availability status of virtual machine in resource group ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512253 | `get_azure_workbooks_details` | ✅ **EXPECTED** | -| 2 | 0.409967 | `edit_azure_workbooks` | ❌ | -| 3 | 0.409085 | `create_azure_workbooks` | ❌ | -| 4 | 0.299382 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.294878 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.421696 | `get_azure_capacity` | ❌ | +| 2 | 0.404977 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.380088 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.377460 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 5 | 0.363367 | `get_azure_signalr_details` | ❌ | --- ## Test 123 -**Expected Tool:** `get_azure_workbooks_details` -**Prompt:** List all workbooks in my resource group +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What metric definitions are available for the Application Insights resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552702 | `get_azure_workbooks_details` | ✅ **EXPECTED** | -| 2 | 0.514558 | `create_azure_workbooks` | ❌ | -| 3 | 0.441697 | `edit_azure_workbooks` | ❌ | -| 4 | 0.426606 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.396091 | `get_azure_security_configurations` | ❌ | +| 1 | 0.519763 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.415402 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.411633 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.410211 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.397417 | `get_azure_capacity` | ❌ | --- ## Test 124 -**Expected Tool:** `get_azure_workbooks_details` -**Prompt:** Show me the workbook with display name +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What resources in resource group have health issues? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.474463 | `get_azure_workbooks_details` | ✅ **EXPECTED** | -| 2 | 0.454790 | `create_azure_workbooks` | ❌ | -| 3 | 0.422535 | `edit_azure_workbooks` | ❌ | -| 4 | 0.201213 | `get_azure_security_configurations` | ❌ | -| 5 | 0.181802 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.505430 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.423077 | `get_azure_capacity` | ❌ | +| 3 | 0.412065 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.389687 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.382850 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 125 -**Expected Tool:** `get_azure_workbooks_details` -**Prompt:** What workbooks do I have in resource group ? +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What service issues have occurred in the last 30 days? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549690 | `get_azure_workbooks_details` | ✅ **EXPECTED** | -| 2 | 0.529693 | `create_azure_workbooks` | ❌ | -| 3 | 0.453173 | `edit_azure_workbooks` | ❌ | -| 4 | 0.438514 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.391845 | `get_azure_security_configurations` | ❌ | +| 1 | 0.274285 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.256635 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.237971 | `get_azure_signalr_details` | ❌ | +| 4 | 0.226131 | `get_azure_capacity` | ❌ | +| 5 | 0.225301 | `get_azure_container_details` | ❌ | --- ## Test 126 -**Expected Tool:** `audit_azure_resources_compliance` -**Prompt:** Check my Azure subscription for any compliance issues or recommendations +**Expected Tool:** `get_azure_resource_and_app_health_status` +**Prompt:** What's the request per second rate for my Application Insights resource over the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546941 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | -| 2 | 0.541006 | `get_azure_capacity` | ❌ | -| 3 | 0.500223 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.477992 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.477388 | `get_azure_best_practices` | ❌ | +| 1 | 0.437195 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.395986 | `get_azure_capacity` | ❌ | +| 3 | 0.371019 | `get_azure_signalr_details` | ❌ | +| 4 | 0.369677 | `create_azure_load_testing` | ❌ | +| 5 | 0.363915 | `create_azure_monitor_webtests` | ❌ | --- ## Test 127 -**Expected Tool:** `audit_azure_resources_compliance` -**Prompt:** Provide compliance recommendations for my current Azure subscription +**Expected Tool:** `deploy_azure_resources_and_applications` +**Prompt:** Create a plan to deploy this application to azure ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.536483 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | -| 2 | 0.511039 | `get_azure_best_practices` | ❌ | -| 3 | 0.490293 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.476949 | `get_azure_capacity` | ❌ | -| 5 | 0.463219 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.640058 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 2 | 0.507616 | `execute_azure_deployments` | ❌ | +| 3 | 0.479918 | `get_azure_best_practices` | ❌ | +| 4 | 0.461377 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.454870 | `add_azure_app_service_database` | ❌ | --- ## Test 128 -**Expected Tool:** `audit_azure_resources_compliance` -**Prompt:** Scan my Azure subscription for compliance recommendations +**Expected Tool:** `deploy_azure_resources_and_applications` +**Prompt:** How can I create a CI/CD pipeline to deploy this app to Azure? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592611 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | -| 2 | 0.508765 | `get_azure_best_practices` | ❌ | -| 3 | 0.492553 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.490633 | `get_azure_capacity` | ❌ | -| 5 | 0.472186 | `execute_azure_cli` | ❌ | +| 1 | 0.578541 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 2 | 0.505622 | `execute_azure_deployments` | ❌ | +| 3 | 0.428397 | `add_azure_app_service_database` | ❌ | +| 4 | 0.410766 | `get_azure_best_practices` | ❌ | +| 5 | 0.372703 | `generate_azure_cli_commands` | ❌ | --- ## Test 129 -**Expected Tool:** `get_azure_security_configurations` -**Prompt:** List all available role assignments in my subscription +**Expected Tool:** `deploy_azure_resources_and_applications` +**Prompt:** Show me the log of the application deployed by azd ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.734098 | `get_azure_security_configurations` | ✅ **EXPECTED** | -| 2 | 0.460328 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.414685 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.368134 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.356299 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.562465 | `execute_azure_deployments` | ❌ | +| 2 | 0.522934 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 3 | 0.442200 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.402454 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.392628 | `get_azure_app_config_settings` | ❌ | --- ## Test 130 -**Expected Tool:** `get_azure_security_configurations` -**Prompt:** Show me the available role assignments in my subscription +**Expected Tool:** `deploy_azure_resources_and_applications` +**Prompt:** Show me the rules to generate bicep scripts ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.702749 | `get_azure_security_configurations` | ✅ **EXPECTED** | -| 2 | 0.485210 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.431017 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.388410 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.380797 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.488008 | `get_azure_best_practices` | ❌ | +| 2 | 0.384841 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 3 | 0.381084 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.325324 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.280777 | `delete_azure_database_admin_configurations` | ❌ | --- ## Test 131 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Display the certificate details for in vault +**Expected Tool:** `execute_azure_deployments` +**Prompt:** Deploy a GPT4o instance on my resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527620 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.508692 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.415809 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.393808 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.326756 | `get_azure_storage_details` | ❌ | +| 1 | 0.307463 | `deploy_azure_resources_and_applications` | ❌ | +| 2 | 0.299302 | `create_azure_load_testing` | ❌ | +| 3 | 0.240425 | `edit_azure_databases` | ❌ | +| 4 | 0.236281 | `get_azure_best_practices` | ❌ | +| 5 | 0.234060 | `execute_azure_deployments` | ✅ **EXPECTED** | --- ## Test 132 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Display the key details for in vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** List all App Configuration stores in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.452262 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.393173 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.342336 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.331707 | `get_azure_storage_details` | ❌ | -| 5 | 0.325902 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.549804 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.418698 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.401422 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.388838 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.378481 | `get_azure_messaging_service_details` | ❌ | --- ## Test 133 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Enumerate certificates in key vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** List all key-value settings in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572328 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.516793 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.473039 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.374233 | `get_azure_security_configurations` | ❌ | -| 5 | 0.320632 | `get_azure_storage_details` | ❌ | +| 1 | 0.605174 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.469735 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.413315 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.323275 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.310582 | `get_azure_database_admin_configuration_details` | ❌ | --- ## Test 134 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Enumerate keys in key vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** List all key-value settings with key name starting with 'prod-' in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.486561 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.456995 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.447167 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.358826 | `get_azure_storage_details` | ❌ | -| 5 | 0.355400 | `get_azure_security_configurations` | ❌ | +| 1 | 0.492475 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.374798 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.344491 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.292927 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.275008 | `get_azure_key_vault_items` | ❌ | --- ## Test 135 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Enumerate secrets in key vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** Show me my App Configuration stores ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508682 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.475020 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.405839 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.352439 | `get_azure_security_configurations` | ❌ | -| 5 | 0.347744 | `get_azure_storage_details` | ❌ | +| 1 | 0.517123 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.397359 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.350017 | `add_azure_app_service_database` | ❌ | +| 4 | 0.318242 | `edit_azure_app_config_settings` | ❌ | +| 5 | 0.300071 | `get_azure_database_admin_configuration_details` | ❌ | --- ## Test 136 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Get the certificate from vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** Show me the App Configuration stores in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.547751 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.477245 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.415517 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.305052 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.273675 | `get_azure_storage_details` | ❌ | +| 1 | 0.564754 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.445478 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.410372 | `add_azure_app_service_database` | ❌ | +| 4 | 0.382377 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.377430 | `get_azure_database_admin_configuration_details` | ❌ | --- ## Test 137 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Get the key from vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** Show me the key-value settings in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.362855 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.355054 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.306974 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.269754 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.240656 | `get_azure_storage_details` | ❌ | +| 1 | 0.619236 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.496884 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.413994 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.320934 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.318437 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 138 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List all certificates in the key vault +**Expected Tool:** `get_azure_app_config_settings` +**Prompt:** Show the content for the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542408 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.534958 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.461491 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.385604 | `get_azure_security_configurations` | ❌ | -| 5 | 0.322461 | `get_azure_storage_details` | ❌ | +| 1 | 0.473674 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.397489 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.351004 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.319847 | `edit_azure_app_config_settings` | ❌ | +| 5 | 0.293222 | `get_azure_key_vault_items` | ❌ | --- ## Test 139 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List all keys in the key vault +**Expected Tool:** `edit_azure_app_config_settings` +**Prompt:** Delete the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502884 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.429278 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.427597 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.378915 | `get_azure_security_configurations` | ❌ | -| 5 | 0.362933 | `get_azure_storage_details` | ❌ | +| 1 | 0.480490 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.419225 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.386233 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.246854 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.245595 | `add_azure_app_service_database` | ❌ | --- ## Test 140 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List all secrets in the key vault +**Expected Tool:** `edit_azure_app_config_settings` +**Prompt:** Set the key in App Configuration store to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502924 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.434446 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.372548 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.348550 | `get_azure_security_configurations` | ❌ | -| 5 | 0.316564 | `get_azure_storage_details` | ❌ | +| 1 | 0.454235 | `lock_unlock_azure_app_config_settings` | ❌ | +| 2 | 0.420016 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.418392 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | +| 4 | 0.282997 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.270549 | `add_azure_app_service_database` | ❌ | --- ## Test 141 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List certificate names in vault +**Expected Tool:** `lock_unlock_azure_app_config_settings` +**Prompt:** Lock the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.522418 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.515660 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.464982 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.392176 | `get_azure_security_configurations` | ❌ | -| 5 | 0.341570 | `get_azure_storage_details` | ❌ | +| 1 | 0.523446 | `lock_unlock_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.367924 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.324653 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.243923 | `add_azure_app_service_database` | ❌ | +| 5 | 0.238376 | `update_azure_managed_lustre_filesystems` | ❌ | --- ## Test 142 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List key names in vault +**Expected Tool:** `lock_unlock_azure_app_config_settings` +**Prompt:** Unlock the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.500762 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.437312 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.409157 | `get_azure_storage_details` | ❌ | -| 4 | 0.407003 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.389283 | `get_azure_security_configurations` | ❌ | +| 1 | 0.552583 | `lock_unlock_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.393938 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.339108 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.276616 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.242957 | `add_azure_app_service_database` | ❌ | --- ## Test 143 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** List secrets names in vault +**Expected Tool:** `edit_azure_workbooks` +**Prompt:** Delete the workbook with resource ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.489437 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.443592 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.381031 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.369698 | `get_azure_storage_details` | ❌ | -| 5 | 0.365721 | `get_azure_security_configurations` | ❌ | +| 1 | 0.505878 | `edit_azure_workbooks` | ✅ **EXPECTED** | +| 2 | 0.375642 | `create_azure_workbooks` | ❌ | +| 3 | 0.362979 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.265457 | `edit_azure_app_config_settings` | ❌ | +| 5 | 0.252814 | `edit_azure_data_analytics_resources` | ❌ | --- ## Test 144 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Retrieve certificate metadata for in vault +**Expected Tool:** `edit_azure_workbooks` +**Prompt:** Update the workbook with a new text step ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.526739 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.500287 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.439402 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.331729 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.303016 | `get_azure_storage_details` | ❌ | +| 1 | 0.496535 | `edit_azure_workbooks` | ✅ **EXPECTED** | +| 2 | 0.413187 | `create_azure_workbooks` | ❌ | +| 3 | 0.327796 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.247622 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.236165 | `update_azure_load_testing_configurations` | ❌ | --- ## Test 145 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Retrieve key metadata for in vault +**Expected Tool:** `create_azure_workbooks` +**Prompt:** Create a new workbook named ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.416058 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.353865 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.345646 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.343080 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.340225 | `get_azure_storage_details` | ❌ | +| 1 | 0.555073 | `create_azure_workbooks` | ✅ **EXPECTED** | +| 2 | 0.400619 | `edit_azure_workbooks` | ❌ | +| 3 | 0.371495 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.225508 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.194912 | `create_azure_key_vault_items` | ❌ | --- ## Test 146 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show certificate names in the key vault +**Expected Tool:** `get_azure_workbooks_details` +**Prompt:** Get information about the workbook with resource ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.557161 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.522419 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.462161 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.379854 | `get_azure_security_configurations` | ❌ | -| 5 | 0.331591 | `get_azure_storage_details` | ❌ | +| 1 | 0.512253 | `get_azure_workbooks_details` | ✅ **EXPECTED** | +| 2 | 0.409967 | `edit_azure_workbooks` | ❌ | +| 3 | 0.409085 | `create_azure_workbooks` | ❌ | +| 4 | 0.299382 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.294878 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 147 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show key names in the key vault +**Expected Tool:** `get_azure_workbooks_details` +**Prompt:** List all workbooks in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508375 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.443413 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.434842 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.402007 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.394387 | `get_azure_storage_details` | ❌ | - ---- - +| 1 | 0.552702 | `get_azure_workbooks_details` | ✅ **EXPECTED** | +| 2 | 0.514558 | `create_azure_workbooks` | ❌ | +| 3 | 0.441697 | `edit_azure_workbooks` | ❌ | +| 4 | 0.426606 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.396091 | `get_azure_security_configurations` | ❌ | + +--- + ## Test 148 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the certificate in the key vault +**Expected Tool:** `get_azure_workbooks_details` +**Prompt:** Show me the workbook with display name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543362 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.500953 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.436563 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.300959 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.298500 | `get_azure_security_configurations` | ❌ | +| 1 | 0.474463 | `get_azure_workbooks_details` | ✅ **EXPECTED** | +| 2 | 0.454790 | `create_azure_workbooks` | ❌ | +| 3 | 0.422536 | `edit_azure_workbooks` | ❌ | +| 4 | 0.201213 | `get_azure_security_configurations` | ❌ | +| 5 | 0.181802 | `browse_azure_marketplace_products` | ❌ | --- ## Test 149 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the certificates in the key vault +**Expected Tool:** `get_azure_workbooks_details` +**Prompt:** What workbooks do I have in resource group ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552928 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.537878 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.461463 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.368299 | `get_azure_security_configurations` | ❌ | -| 5 | 0.330062 | `get_azure_storage_details` | ❌ | +| 1 | 0.549690 | `get_azure_workbooks_details` | ✅ **EXPECTED** | +| 2 | 0.529693 | `create_azure_workbooks` | ❌ | +| 3 | 0.453173 | `edit_azure_workbooks` | ❌ | +| 4 | 0.438514 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.391845 | `get_azure_security_configurations` | ❌ | --- ## Test 150 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the details of the certificate in the key vault +**Expected Tool:** `audit_azure_resources_compliance` +**Prompt:** Check my Azure subscription for any compliance issues or recommendations ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.514998 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.490672 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.420402 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.409324 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.358500 | `get_azure_storage_details` | ❌ | +| 1 | 0.546941 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | +| 2 | 0.541006 | `get_azure_capacity` | ❌ | +| 3 | 0.500223 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.477388 | `get_azure_best_practices` | ❌ | +| 5 | 0.474918 | `get_azure_resource_and_app_health_status` | ❌ | --- ## Test 151 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the details of the key in the key vault +**Expected Tool:** `audit_azure_resources_compliance` +**Prompt:** Provide compliance recommendations for my current Azure subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.467928 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.426201 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.377132 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.374666 | `get_azure_storage_details` | ❌ | -| 5 | 0.367478 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.536483 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | +| 2 | 0.511039 | `get_azure_best_practices` | ❌ | +| 3 | 0.490293 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.476949 | `get_azure_capacity` | ❌ | +| 5 | 0.463219 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 152 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the key in the key vault +**Expected Tool:** `audit_azure_resources_compliance` +**Prompt:** Scan my Azure subscription for compliance recommendations ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.436642 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.399191 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.371997 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.301224 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.286946 | `get_azure_storage_details` | ❌ | +| 1 | 0.592611 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | +| 2 | 0.508765 | `get_azure_best_practices` | ❌ | +| 3 | 0.492553 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.490633 | `get_azure_capacity` | ❌ | +| 5 | 0.455630 | `deploy_azure_resources_and_applications` | ❌ | --- ## Test 153 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the keys in the key vault +**Expected Tool:** `generate_azure_cli_commands` +**Prompt:** Get Azure CLI command to create a Storage account with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.499555 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.448371 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.432123 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.344132 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.341486 | `get_azure_storage_details` | ❌ | +| 1 | 0.559532 | `create_azure_storage` | ❌ | +| 2 | 0.490462 | `get_azure_storage_details` | ❌ | +| 3 | 0.486254 | `generate_azure_cli_commands` | ✅ **EXPECTED** | +| 4 | 0.428312 | `install_azure_cli_extensions` | ❌ | +| 5 | 0.417789 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 154 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show me the secrets in the key vault +**Expected Tool:** `generate_azure_cli_commands` +**Prompt:** Show me how to use Azure CLI to list all virtual machines in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530658 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.461601 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.426648 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.367244 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.360507 | `get_azure_storage_details` | ❌ | +| 1 | 0.505461 | `generate_azure_cli_commands` | ✅ **EXPECTED** | +| 2 | 0.466697 | `install_azure_cli_extensions` | ❌ | +| 3 | 0.460094 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.439158 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.433504 | `get_azure_virtual_desktop_details` | ❌ | --- ## Test 155 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** Show secrets names in the key vault +**Expected Tool:** `generate_azure_cli_commands` +**Prompt:** Show me the details of the storage account with Azure CLI commands ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.505159 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.453385 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.427298 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.356917 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.339292 | `get_azure_storage_details` | ❌ | +| 1 | 0.612935 | `get_azure_storage_details` | ❌ | +| 2 | 0.497121 | `generate_azure_cli_commands` | ✅ **EXPECTED** | +| 3 | 0.475109 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.471795 | `create_azure_storage` | ❌ | +| 5 | 0.455795 | `get_azure_container_details` | ❌ | --- ## Test 156 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** What certificates are in the key vault ? +**Expected Tool:** `get_azure_security_configurations` +**Prompt:** List all available role assignments in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.570872 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.566285 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 3 | 0.542657 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.354249 | `get_azure_storage_details` | ❌ | -| 5 | 0.338482 | `get_azure_security_configurations` | ❌ | +| 1 | 0.734114 | `get_azure_security_configurations` | ✅ **EXPECTED** | +| 2 | 0.460374 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.414713 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.356351 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.341765 | `get_azure_messaging_service_details` | ❌ | --- ## Test 157 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** What keys are in the key vault ? +**Expected Tool:** `get_azure_security_configurations` +**Prompt:** Show me the available role assignments in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532438 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.517834 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.484768 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.375040 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.371608 | `get_azure_storage_details` | ❌ | +| 1 | 0.702749 | `get_azure_security_configurations` | ✅ **EXPECTED** | +| 2 | 0.485211 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.431017 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.388410 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.357972 | `get_azure_capacity` | ❌ | --- ## Test 158 -**Expected Tool:** `get_azure_key_vault` -**Prompt:** What secrets are in the key vault ? +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Display the certificate details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.545085 | `get_azure_key_vault` | ✅ **EXPECTED** | -| 2 | 0.520862 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.438877 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.412144 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.401361 | `get_azure_storage_details` | ❌ | +| 1 | 0.533833 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.508789 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.465713 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.426342 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.393933 | `get_azure_app_config_settings` | ❌ | --- ## Test 159 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Add a new version of secret with value in vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Display the key details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.504694 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.391544 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.340089 | `get_azure_key_vault` | ❌ | -| 4 | 0.337182 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.315313 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.459981 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.447387 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.393272 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.343733 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.342376 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 160 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create a new certificate called in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Enumerate certificates in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577228 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.536615 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.403721 | `get_azure_key_vault` | ❌ | -| 4 | 0.283179 | `create_azure_storage` | ❌ | -| 5 | 0.282142 | `create_azure_workbooks` | ❌ | +| 1 | 0.572193 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.513646 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.495222 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.438478 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.423154 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 161 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create a new key called with the RSA type in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Enumerate keys in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.494654 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.417922 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.335176 | `get_azure_key_vault` | ❌ | -| 4 | 0.282042 | `create_azure_storage` | ❌ | -| 5 | 0.231202 | `create_azure_workbooks` | ❌ | +| 1 | 0.501656 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.492799 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.477019 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.466847 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.447167 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 162 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create a new secret called with value in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Enumerate secrets in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.551337 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.385343 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.367275 | `get_azure_key_vault` | ❌ | -| 4 | 0.301886 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.294044 | `create_azure_storage` | ❌ | +| 1 | 0.614347 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.522680 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.504917 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.450082 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.405839 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 163 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create an EC key with name in the vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Get the account settings for my key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.455389 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.407173 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.338543 | `get_azure_key_vault` | ❌ | -| 4 | 0.274881 | `create_azure_storage` | ❌ | -| 5 | 0.232171 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.510037 | `get_azure_app_config_settings` | ❌ | +| 2 | 0.469247 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.446870 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.391544 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.377218 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 164 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create an oct key in the vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Get the certificate from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502036 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.450939 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.385868 | `get_azure_key_vault` | ❌ | -| 4 | 0.301799 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.289852 | `create_azure_workbooks` | ❌ | +| 1 | 0.547751 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.482982 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.476094 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.435609 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.378696 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 165 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Create an RSA key in the vault with name +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Get the key from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.519876 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.437054 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.366580 | `get_azure_key_vault` | ❌ | -| 4 | 0.280091 | `create_azure_storage` | ❌ | -| 5 | 0.249914 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.424659 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.371826 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.356181 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.355054 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.315829 | `create_azure_key_vault_items` | ❌ | --- ## Test 166 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Generate a certificate named in key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List all certificates in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.534280 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.530813 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.417740 | `get_azure_key_vault` | ❌ | -| 4 | 0.282987 | `create_azure_storage` | ❌ | -| 5 | 0.270041 | `audit_azure_resources_compliance` | ❌ | +| 1 | 0.542408 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.529198 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.485747 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.425943 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.418508 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 167 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Generate a key with type in vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List all keys in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.460685 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.387664 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.343094 | `get_azure_key_vault` | ❌ | -| 4 | 0.272969 | `create_azure_storage` | ❌ | -| 5 | 0.243119 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.503471 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.468167 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.459043 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.442074 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.427597 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 168 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Issue a certificate in key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List all secrets in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.559527 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.521991 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.440867 | `get_azure_key_vault` | ❌ | -| 4 | 0.277842 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.277213 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.580241 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.494605 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.490556 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.413675 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.372548 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 169 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Provision a new key vault certificate in vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List certificate names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555685 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.527468 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.396987 | `get_azure_key_vault` | ❌ | -| 4 | 0.264595 | `create_azure_storage` | ❌ | -| 5 | 0.263716 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.526271 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.522418 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.482074 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.447164 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.435835 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 170 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Request creation of certificate in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List key names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.545314 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.540640 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.428272 | `get_azure_key_vault` | ❌ | -| 4 | 0.286085 | `create_azure_storage` | ❌ | -| 5 | 0.283203 | `create_azure_workbooks` | ❌ | +| 1 | 0.520132 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.475321 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.465846 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.441517 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.407002 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 171 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Set a secret named with value in key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** List secrets names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.439305 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.357475 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.356853 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.347919 | `get_azure_key_vault` | ❌ | -| 5 | 0.347006 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.545546 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.497984 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.490924 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.430161 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.381031 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 172 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Store secret value in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Retrieve certificate metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.476932 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.402382 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.372842 | `get_azure_key_vault` | ❌ | -| 4 | 0.329065 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.301220 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.526748 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.521132 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.458697 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.449969 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.352703 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 173 -**Expected Tool:** `create_azure_key_vault_items` -**Prompt:** Update secret to value in the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Retrieve key metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.411404 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.365562 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.356877 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.352415 | `get_azure_key_vault` | ❌ | -| 5 | 0.342010 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.448461 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.440374 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.360597 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.347427 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.345646 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 174 -**Expected Tool:** `import_azure_key_vault_certificates` -**Prompt:** Add existing certificate file to the key vault with name +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show certificate names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.560409 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.430852 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.339090 | `get_azure_key_vault` | ❌ | -| 4 | 0.221680 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.216397 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.557161 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.535803 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.480958 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.470813 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.436295 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 175 -**Expected Tool:** `import_azure_key_vault_certificates` -**Prompt:** Import a certificate into the key vault using the name +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show key names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.660982 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.459787 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.378360 | `get_azure_key_vault` | ❌ | -| 4 | 0.256701 | `deploy_azure_ai_models` | ❌ | -| 5 | 0.240543 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.532996 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.527983 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.476688 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.443413 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.438251 | `create_azure_key_vault_items` | ❌ | --- ## Test 176 -**Expected Tool:** `import_azure_key_vault_certificates` -**Prompt:** Import the certificate in file into the key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the account settings for managed HSM keyvault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.645826 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.425682 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.376495 | `get_azure_key_vault` | ❌ | -| 4 | 0.249209 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.248738 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.459394 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.434426 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.393297 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.376496 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.354255 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 177 -**Expected Tool:** `import_azure_key_vault_certificates` -**Prompt:** Load certificate from file into vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630975 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.443601 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.422833 | `get_azure_key_vault` | ❌ | -| 4 | 0.233437 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.222987 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.543492 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.508166 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.454602 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.453808 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.386522 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 178 -**Expected Tool:** `import_azure_key_vault_certificates` -**Prompt:** Upload certificate file to key vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the certificates in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.587412 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.451230 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.396327 | `get_azure_key_vault` | ❌ | -| 4 | 0.334084 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.287607 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.552937 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.535104 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.480107 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.475447 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.430616 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 179 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Fetch the Azure Terraform best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the details of the certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.735075 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.524574 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.474628 | `search_microsoft_docs` | ❌ | -| 4 | 0.459823 | `execute_azure_cli` | ❌ | -| 5 | 0.437042 | `get_azure_capacity` | ❌ | +| 1 | 0.527499 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.490655 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.460251 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.432964 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.409334 | `get_azure_app_config_settings` | ❌ | --- ## Test 180 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the details of the key in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.690117 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.601714 | `search_microsoft_docs` | ❌ | -| 3 | 0.539669 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.508718 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.483779 | `get_azure_capacity` | ❌ | +| 1 | 0.480327 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.471396 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.426126 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.395013 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.377245 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 181 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure code generation best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the key in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.713406 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.543696 | `search_microsoft_docs` | ❌ | -| 3 | 0.529617 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.470109 | `design_azure_architecture` | ❌ | -| 5 | 0.435613 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.464726 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.449853 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.407815 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.399191 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.379916 | `create_azure_key_vault_items` | ❌ | --- ## Test 182 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure deployment best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the keys in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.683437 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.614180 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.558589 | `search_microsoft_docs` | ❌ | -| 4 | 0.465496 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.450131 | `get_azure_capacity` | ❌ | +| 1 | 0.517414 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.502886 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.480193 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.448371 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.441572 | `create_azure_key_vault_items` | ❌ | --- ## Test 183 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure Functions best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show me the secrets in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.682026 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.556022 | `search_microsoft_docs` | ❌ | -| 3 | 0.509047 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.505735 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.443358 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.587359 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.574899 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.526650 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.440536 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.426022 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 184 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure Functions code generation best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** Show secrets names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.685214 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.486075 | `search_microsoft_docs` | ❌ | -| 3 | 0.480287 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.448692 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.416416 | `execute_azure_developer_cli` | ❌ | +| 1 | 0.595761 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.581393 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.517120 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.434361 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.427298 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 185 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure Functions deployment best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** What certificates are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.675358 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.571007 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.527827 | `search_microsoft_docs` | ❌ | -| 4 | 0.497925 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.435563 | `deploy_azure_ai_models` | ❌ | +| 1 | 0.583734 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.570872 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.555201 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.477077 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.476712 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 186 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Get the latest Azure Static Web Apps best practices +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** What keys are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612873 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.520938 | `search_microsoft_docs` | ❌ | -| 3 | 0.518435 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.424667 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.421748 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.551081 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.543959 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.534076 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.520839 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.484809 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 187 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** How can I use Bicep to create an Azure OpenAI service? +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** What secrets are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.489465 | `deploy_resources_and_applications_to_azure` | ❌ | -| 2 | 0.480742 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 3 | 0.440374 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.436599 | `deploy_azure_ai_models` | ❌ | -| 5 | 0.432577 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 1 | 0.622269 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.581377 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.564774 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.495588 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.438784 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 188 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** Show me the Azure Terraform best practices and generate code sample to get a secret from Azure Key Vault +**Expected Tool:** `get_azure_key_vault_items` +**Prompt:** What's the value of the setting in my key vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.618224 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.489003 | `get_azure_key_vault` | ❌ | -| 3 | 0.478352 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.444083 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.435792 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.498669 | `get_azure_app_config_settings` | ❌ | +| 2 | 0.468556 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.448361 | `lock_unlock_azure_app_config_settings` | ❌ | +| 4 | 0.444399 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 5 | 0.409423 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 189 -**Expected Tool:** `get_azure_best_practices` -**Prompt:** What are azure function best practices? +**Expected Tool:** `get_azure_key_vault_secret_values` +**Prompt:** Display the secret details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.628027 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.454934 | `get_azure_app_resource_details` | ❌ | -| 3 | 0.453910 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.451968 | `search_microsoft_docs` | ❌ | -| 5 | 0.391204 | `execute_azure_cli` | ❌ | +| 1 | 0.586421 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.526749 | `get_azure_key_vault_items` | ❌ | +| 3 | 0.517989 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.440736 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.389603 | `create_azure_key_vault_items` | ❌ | --- ## Test 190 -**Expected Tool:** `design_azure_architecture` -**Prompt:** Generate the azure architecture diagram for this application +**Expected Tool:** `get_azure_key_vault_secret_values` +**Prompt:** Get the secret from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.676638 | `design_azure_architecture` | ✅ **EXPECTED** | -| 2 | 0.481643 | `get_azure_best_practices` | ❌ | -| 3 | 0.465832 | `deploy_resources_and_applications_to_azure` | ❌ | -| 4 | 0.386867 | `execute_azure_developer_cli` | ❌ | -| 5 | 0.385801 | `audit_azure_resources_compliance` | ❌ | +| 1 | 0.563408 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.504391 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.450877 | `get_azure_key_vault_items` | ❌ | +| 4 | 0.381027 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.364969 | `create_azure_key_vault_items` | ❌ | --- ## Test 191 -**Expected Tool:** `design_azure_architecture` -**Prompt:** Help me create a cloud service that will serve as ATM for users +**Expected Tool:** `get_azure_key_vault_secret_values` +**Prompt:** Retrieve secret metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.259518 | `browse_azure_marketplace_products` | ❌ | -| 2 | 0.248253 | `design_azure_architecture` | ✅ **EXPECTED** | -| 3 | 0.241293 | `create_azure_storage` | ❌ | -| 4 | 0.229913 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.226031 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.558674 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.487736 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.486559 | `get_azure_key_vault_items` | ❌ | +| 4 | 0.389038 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.365020 | `get_azure_app_config_settings` | ❌ | --- ## Test 192 -**Expected Tool:** `design_azure_architecture` -**Prompt:** How can I design a cloud service in Azure that will store and present videos for users? +**Expected Tool:** `get_azure_key_vault_secret_values` +**Prompt:** Show me the details of the secret in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.427652 | `upload_azure_storage_blobs` | ❌ | -| 2 | 0.413253 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.410459 | `create_azure_storage` | ❌ | -| 4 | 0.410342 | `search_microsoft_docs` | ❌ | -| 5 | 0.405913 | `design_azure_architecture` | ✅ **EXPECTED** | +| 1 | 0.542793 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.507689 | `get_azure_key_vault_items` | ❌ | +| 3 | 0.493634 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.432729 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.375435 | `create_azure_key_vault_items` | ❌ | --- ## Test 193 -**Expected Tool:** `design_azure_architecture` -**Prompt:** I want to design a cloud app for ordering groceries +**Expected Tool:** `get_azure_key_vault_secret_values` +**Prompt:** Show me the secret in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.342485 | `browse_azure_marketplace_products` | ❌ | -| 2 | 0.289308 | `design_azure_architecture` | ✅ **EXPECTED** | -| 3 | 0.277705 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.276204 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.224286 | `get_azure_best_practices` | ❌ | +| 1 | 0.552814 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.516089 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.487940 | `get_azure_key_vault_items` | ❌ | +| 4 | 0.389282 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.386391 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 194 -**Expected Tool:** `design_azure_architecture` -**Prompt:** Please help me design an architecture for a large-scale file upload, storage, and retrieval service +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Create a new certificate called in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.323311 | `upload_azure_storage_blobs` | ❌ | -| 2 | 0.296166 | `design_azure_architecture` | ✅ **EXPECTED** | -| 3 | 0.245346 | `create_azure_storage` | ❌ | -| 4 | 0.224135 | `get_azure_capacity` | ❌ | -| 5 | 0.207400 | `get_azure_best_practices` | ❌ | +| 1 | 0.602182 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.536615 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.536205 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.416317 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.323442 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 195 -**Expected Tool:** `get_azure_load_testing_details` -**Prompt:** Get all the load test runs for the test with id in the load test resource in resource group +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Create a new key called with the RSA type in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609585 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 2 | 0.568056 | `create_azure_load_testing` | ❌ | -| 3 | 0.448055 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.366497 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.329986 | `get_azure_storage_details` | ❌ | +| 1 | 0.527822 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.506017 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.417486 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.354703 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.330867 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 196 -**Expected Tool:** `get_azure_load_testing_details` -**Prompt:** Get the load test run with id in the load test resource in resource group +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Create an EC key with name in the vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.599651 | `create_azure_load_testing` | ❌ | -| 2 | 0.581081 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 3 | 0.457483 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.357813 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.321301 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.492989 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.470079 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.407173 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.350415 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.338214 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 197 -**Expected Tool:** `get_azure_load_testing_details` -**Prompt:** Get the load test with id in the load test resource in resource group +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Create an oct key in the vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612800 | `create_azure_load_testing` | ❌ | -| 2 | 0.592725 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 3 | 0.421873 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.349117 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.333908 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.560056 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.506522 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.450938 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.422702 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.392705 | `get_azure_key_vault_items` | ❌ | --- ## Test 198 -**Expected Tool:** `get_azure_load_testing_details` -**Prompt:** List all load testing resources in the resource group in my subscription +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Create an RSA key in the vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.669717 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 2 | 0.609875 | `create_azure_load_testing` | ❌ | -| 3 | 0.493520 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.421963 | `get_azure_capacity` | ❌ | -| 5 | 0.411835 | `get_azure_storage_details` | ❌ | +| 1 | 0.545276 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.534559 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.437082 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.391019 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.372312 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 199 -**Expected Tool:** `create_azure_load_testing` -**Prompt:** Create a basic URL test using the following endpoint URL that runs for 30 minutes with 45 virtual users. The test name is with the test id and the load testing resource is in the resource group in my subscription +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Generate a certificate named in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542817 | `create_azure_load_testing` | ✅ **EXPECTED** | -| 2 | 0.431906 | `get_azure_load_testing_details` | ❌ | -| 3 | 0.425527 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.328438 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.303424 | `create_azure_workbooks` | ❌ | +| 1 | 0.559592 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.530817 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.487915 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.426824 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.367947 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 200 -**Expected Tool:** `create_azure_load_testing` -**Prompt:** Create a load test resource in the resource group in my subscription +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Generate a key with type in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.659823 | `create_azure_load_testing` | ✅ **EXPECTED** | -| 2 | 0.529798 | `get_azure_load_testing_details` | ❌ | -| 3 | 0.410977 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.374224 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.334648 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.484318 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.468180 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.387664 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.356823 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.351880 | `get_azure_key_vault_items` | ❌ | --- ## Test 201 -**Expected Tool:** `create_azure_load_testing` -**Prompt:** Create a test run using the id for test in the load testing resource in resource group . Use the name of test run and description as +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Issue a certificate in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.585612 | `create_azure_load_testing` | ✅ **EXPECTED** | -| 2 | 0.496772 | `update_azure_load_testing_configurations` | ❌ | -| 3 | 0.460907 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.319822 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.297908 | `deploy_resources_and_applications_to_azure` | ❌ | +| 1 | 0.559527 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.546365 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.476974 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.452962 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.372106 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 202 -**Expected Tool:** `update_azure_load_testing_configurations` -**Prompt:** Update a test run display name as for the id for test in the load testing resource in resource group . +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Provision a new key vault certificate in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577419 | `update_azure_load_testing_configurations` | ✅ **EXPECTED** | -| 2 | 0.501316 | `create_azure_load_testing` | ❌ | -| 3 | 0.443800 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.303358 | `edit_azure_workbooks` | ❌ | -| 5 | 0.257550 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 1 | 0.555686 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.554286 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.500237 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.410230 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.325473 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 203 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Get the schema configuration for knowledge index +**Expected Tool:** `create_azure_key_vault_items` +**Prompt:** Request creation of certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.310095 | `get_azure_app_config_settings` | ❌ | -| 2 | 0.294949 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.268668 | `get_azure_best_practices` | ❌ | -| 4 | 0.262165 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.249688 | `get_azure_workbooks_details` | ❌ | +| 1 | 0.563546 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.545314 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.490569 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.439216 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.343811 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 204 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** List all AI Foundry model deployments +**Expected Tool:** `create_azure_key_vault_secrets` +**Prompt:** Add a new version of secret with value in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619823 | `deploy_azure_ai_models` | ❌ | -| 2 | 0.575424 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.474409 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.412586 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.359607 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.629763 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.479387 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.407897 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.391544 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.363090 | `append_azure_confidential_ledger_entries` | ❌ | --- ## Test 205 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** List all AI Foundry models +**Expected Tool:** `create_azure_key_vault_secrets` +**Prompt:** Create a new secret called with value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517204 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.472873 | `deploy_azure_ai_models` | ❌ | -| 3 | 0.424302 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.367885 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.338517 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.674434 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.528671 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.442879 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.386205 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.385343 | `import_azure_key_vault_certificates` | ❌ | --- ## Test 206 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** List all Cognitive Search services in my subscription +**Expected Tool:** `create_azure_key_vault_secrets` +**Prompt:** Set a secret named with value in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530525 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.474685 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.444086 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.399309 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.391196 | `get_azure_security_configurations` | ❌ | +| 1 | 0.572358 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.454589 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.415391 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.387428 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.357411 | `lock_unlock_azure_app_config_settings` | ❌ | --- ## Test 207 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** List all indexes in the Cognitive Search service +**Expected Tool:** `create_azure_key_vault_secrets` +**Prompt:** Store secret value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.482355 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.356614 | `get_azure_security_configurations` | ❌ | -| 3 | 0.350774 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.349687 | `get_azure_databases_details` | ❌ | -| 5 | 0.343922 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.588795 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.475056 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.452602 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.402382 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.395541 | `get_azure_key_vault_items` | ❌ | --- ## Test 208 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** List all knowledge indexes in my AI Foundry project +**Expected Tool:** `create_azure_key_vault_secrets` +**Prompt:** Update secret to value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.522175 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.426054 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.407003 | `deploy_azure_ai_models` | ❌ | -| 4 | 0.330976 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.288627 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.514668 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.453316 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.390764 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.376214 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.365562 | `lock_unlock_azure_app_config_settings` | ❌ | --- ## Test 209 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Search for instances of in the index in Cognitive Search service +**Expected Tool:** `import_azure_key_vault_certificates` +**Prompt:** Add existing certificate file to the key vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.426451 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.296542 | `get_azure_resource_and_app_health_status` | ❌ | -| 3 | 0.280948 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.278893 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.268471 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 1 | 0.560409 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.451439 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.389225 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.351729 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.275281 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 210 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me all AI Foundry model deployments +**Expected Tool:** `import_azure_key_vault_certificates` +**Prompt:** Import a certificate into the key vault using the name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612664 | `deploy_azure_ai_models` | ❌ | -| 2 | 0.565595 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.496487 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.425410 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.389499 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.660982 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.486040 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.420000 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.383841 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.352274 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 211 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me my Cognitive Search services +**Expected Tool:** `import_azure_key_vault_certificates` +**Prompt:** Import the certificate in file into the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.485141 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.431878 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.370196 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.362364 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.357646 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.645826 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.444963 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.405491 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.367234 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.360848 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 212 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the available AI Foundry models +**Expected Tool:** `import_azure_key_vault_certificates` +**Prompt:** Load certificate from file into vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533639 | `deploy_azure_ai_models` | ❌ | -| 2 | 0.529802 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.499511 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.408332 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.384895 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.630975 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.464828 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.419529 | `get_azure_key_vault_items` | ❌ | +| 4 | 0.390232 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.388920 | `create_azure_key_vault_secrets` | ❌ | --- ## Test 213 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the Cognitive Search services in my subscription +**Expected Tool:** `import_azure_key_vault_certificates` +**Prompt:** Upload certificate file to key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.541256 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.507116 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.458382 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.414408 | `get_azure_capacity` | ❌ | -| 5 | 0.413622 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.587413 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.473576 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.423146 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.395735 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.362752 | `get_azure_key_vault_secret_values` | ❌ | --- ## Test 214 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the details of the index in Cognitive Search service +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Fetch the Azure Terraform best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.509173 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.430939 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.406409 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.398647 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.378065 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.735005 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.524465 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.439480 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.436925 | `get_azure_capacity` | ❌ | +| 5 | 0.436798 | `browse_azure_marketplace_products` | ❌ | --- ## Test 215 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the indexes in the Cognitive Search service +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.472474 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.362349 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.361055 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.349980 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.345299 | `get_azure_databases_details` | ❌ | +| 1 | 0.690117 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.539669 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.508718 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.483779 | `get_azure_capacity` | ❌ | +| 5 | 0.460726 | `get_azure_storage_details` | ❌ | --- ## Test 216 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the knowledge indexes in my AI Foundry project +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure code generation best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.511182 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.474227 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.414405 | `deploy_azure_ai_models` | ❌ | -| 4 | 0.337209 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.316338 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.713406 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.529617 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.470109 | `design_azure_architecture` | ❌ | +| 4 | 0.437561 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.435613 | `browse_azure_marketplace_products` | ❌ | --- ## Test 217 -**Expected Tool:** `get_azure_ai_resources_details` -**Prompt:** Show me the schema for knowledge index in my AI Foundry project +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure deployment best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498412 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.373160 | `deploy_azure_ai_models` | ❌ | -| 3 | 0.341967 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.324788 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.308722 | `get_azure_best_practices` | ❌ | +| 1 | 0.683437 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.614180 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.475963 | `execute_azure_deployments` | ❌ | +| 4 | 0.465496 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.450131 | `get_azure_capacity` | ❌ | --- ## Test 218 -**Expected Tool:** `deploy_azure_ai_models` -**Prompt:** Deploy a GPT4o instance on my resource +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure Functions best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.387447 | `deploy_azure_ai_models` | ✅ **EXPECTED** | -| 2 | 0.307463 | `deploy_resources_and_applications_to_azure` | ❌ | -| 3 | 0.299302 | `create_azure_load_testing` | ❌ | -| 4 | 0.240425 | `edit_azure_databases` | ❌ | -| 5 | 0.236281 | `get_azure_best_practices` | ❌ | +| 1 | 0.682026 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.557930 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.505735 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.443359 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.431202 | `get_azure_capacity` | ❌ | --- ## Test 219 -**Expected Tool:** `connect_azure_ai_foundry_agents` -**Prompt:** Query an agent in my AI foundry project +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure Functions code generation best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.561982 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 2 | 0.514602 | `connect_azure_ai_foundry_agents` | ✅ **EXPECTED** | -| 3 | 0.452053 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.382785 | `deploy_azure_ai_models` | ❌ | -| 5 | 0.282804 | `execute_azure_cli` | ❌ | +| 1 | 0.685214 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.499419 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.480287 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.425068 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.410038 | `execute_azure_deployments` | ❌ | --- ## Test 220 -**Expected Tool:** `evaluate_azure_ai_foundry_agents` -**Prompt:** Evaluate the full query and response I got from my agent for task_adherence +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure Functions deployment best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.347263 | `evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | -| 2 | 0.280105 | `get_azure_resource_and_app_health_status` | ❌ | -| 3 | 0.245804 | `execute_azure_cli` | ❌ | -| 4 | 0.238878 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.229140 | `audit_azure_resources_compliance` | ❌ | +| 1 | 0.675326 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.570982 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.537910 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.460530 | `execute_azure_deployments` | ❌ | +| 5 | 0.415282 | `browse_azure_marketplace_products` | ❌ | --- ## Test 221 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Get details about the storage account +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Get the latest Azure Static Web Apps best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.627423 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.475469 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.449003 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.429033 | `create_azure_storage` | ❌ | -| 5 | 0.412908 | `get_azure_container_details` | ❌ | +| 1 | 0.612873 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.518435 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.424667 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.418814 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.417729 | `execute_azure_deployments` | ❌ | --- ## Test 222 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Get the details about blob in the container in storage account +**Expected Tool:** `get_azure_best_practices` +**Prompt:** How can I use Bicep to create an Azure OpenAI service? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.580769 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.529061 | `create_azure_storage` | ❌ | -| 3 | 0.478682 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.448552 | `get_azure_container_details` | ❌ | -| 5 | 0.415661 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.489465 | `deploy_azure_resources_and_applications` | ❌ | +| 2 | 0.486962 | `use_azure_openai_models` | ❌ | +| 3 | 0.480742 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 4 | 0.451552 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.441313 | `get_azure_ai_resources_details` | ❌ | --- ## Test 223 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List all blob containers in the storage account +**Expected Tool:** `get_azure_best_practices` +**Prompt:** Show me the Azure Terraform best practices and generate code sample to get a secret from Azure Key Vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.587289 | `create_azure_storage` | ❌ | -| 2 | 0.514466 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 3 | 0.453206 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.357190 | `get_azure_container_details` | ❌ | -| 5 | 0.336886 | `get_azure_security_configurations` | ❌ | +| 1 | 0.618330 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.535723 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.531963 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.465939 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.462316 | `create_azure_key_vault_items` | ❌ | --- ## Test 224 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List all blobs in the blob container in the storage account +**Expected Tool:** `get_azure_best_practices` +**Prompt:** What are azure function best practices? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.551075 | `create_azure_storage` | ❌ | -| 2 | 0.493746 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 3 | 0.463728 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.338653 | `get_azure_container_details` | ❌ | -| 5 | 0.309756 | `get_azure_security_configurations` | ❌ | +| 1 | 0.628027 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.475164 | `get_azure_app_resource_details` | ❌ | +| 3 | 0.453910 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.380736 | `execute_azure_deployments` | ❌ | +| 5 | 0.379701 | `get_azure_capacity` | ❌ | --- ## Test 225 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List all storage accounts in my subscription including their location and SKU +**Expected Tool:** `design_azure_architecture` +**Prompt:** Generate the azure architecture diagram for this application ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.599918 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.459444 | `create_azure_storage` | ❌ | -| 3 | 0.444366 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.428351 | `get_azure_capacity` | ❌ | -| 5 | 0.417674 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.676638 | `design_azure_architecture` | ✅ **EXPECTED** | +| 2 | 0.481643 | `get_azure_best_practices` | ❌ | +| 3 | 0.465832 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.406230 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.403087 | `generate_azure_cli_commands` | ❌ | --- ## Test 226 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List the Azure Managed Lustre filesystems in my resource group +**Expected Tool:** `design_azure_architecture` +**Prompt:** Help me create a cloud service that will serve as ATM for users ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.575069 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.432148 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.421195 | `get_azure_capacity` | ❌ | -| 4 | 0.408405 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.389895 | `get_azure_databases_details` | ❌ | +| 1 | 0.269652 | `create_azure_storage` | ❌ | +| 2 | 0.259518 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.248253 | `design_azure_architecture` | ✅ **EXPECTED** | +| 4 | 0.244271 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.231667 | `create_azure_sql_databases_and_servers` | ❌ | --- ## Test 227 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List the Azure Managed Lustre filesystems in my subscription +**Expected Tool:** `design_azure_architecture` +**Prompt:** How can I design a cloud service in Azure that will store and present videos for users? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572276 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.455172 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.431655 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.422149 | `get_azure_capacity` | ❌ | -| 5 | 0.400784 | `get_azure_databases_details` | ❌ | +| 1 | 0.427652 | `upload_azure_storage_blobs` | ❌ | +| 2 | 0.424080 | `create_azure_storage` | ❌ | +| 3 | 0.418547 | `get_azure_storage_details` | ❌ | +| 4 | 0.413253 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.405913 | `design_azure_architecture` | ✅ **EXPECTED** | --- ## Test 228 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** List the Azure Managed Lustre SKUs available in +**Expected Tool:** `design_azure_architecture` +**Prompt:** I want to design a cloud app for ordering groceries ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600781 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.492224 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.455935 | `get_azure_capacity` | ❌ | -| 4 | 0.404690 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.390248 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.342485 | `browse_azure_marketplace_products` | ❌ | +| 2 | 0.289308 | `design_azure_architecture` | ✅ **EXPECTED** | +| 3 | 0.276203 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.251655 | `add_azure_app_service_database` | ❌ | +| 5 | 0.236505 | `execute_azure_deployments` | ❌ | --- ## Test 229 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me my storage accounts with whether hierarchical namespace (HNS) is enabled +**Expected Tool:** `design_azure_architecture` +**Prompt:** Please help me design an architecture for a large-scale file upload, storage, and retrieval service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.506913 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.404888 | `create_azure_storage` | ❌ | -| 3 | 0.388202 | `get_azure_capacity` | ❌ | -| 4 | 0.384632 | `get_azure_security_configurations` | ❌ | -| 5 | 0.356058 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.323447 | `upload_azure_storage_blobs` | ❌ | +| 2 | 0.305869 | `create_azure_storage` | ❌ | +| 3 | 0.296671 | `design_azure_architecture` | ✅ **EXPECTED** | +| 4 | 0.261376 | `get_azure_storage_details` | ❌ | +| 5 | 0.235389 | `update_azure_managed_lustre_filesystems` | ❌ | --- ## Test 230 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the blobs in the blob container in the storage account +**Expected Tool:** `get_azure_load_testing_details` +**Prompt:** Get all the load test runs for the test with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546896 | `create_azure_storage` | ❌ | -| 2 | 0.483715 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 3 | 0.472639 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.397280 | `get_azure_container_details` | ❌ | -| 5 | 0.309579 | `get_azure_security_configurations` | ❌ | +| 1 | 0.609585 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 2 | 0.568056 | `create_azure_load_testing` | ❌ | +| 3 | 0.448056 | `update_azure_load_testing_configurations` | ❌ | +| 4 | 0.366497 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.353481 | `create_azure_monitor_webtests` | ❌ | --- ## Test 231 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the containers in the storage account +**Expected Tool:** `get_azure_load_testing_details` +**Prompt:** Get the load test run with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543409 | `create_azure_storage` | ❌ | -| 2 | 0.520343 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 3 | 0.418173 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.408752 | `get_azure_container_details` | ❌ | -| 5 | 0.354495 | `get_azure_security_configurations` | ❌ | +| 1 | 0.599651 | `create_azure_load_testing` | ❌ | +| 2 | 0.581081 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 3 | 0.457483 | `update_azure_load_testing_configurations` | ❌ | +| 4 | 0.357813 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.350739 | `create_azure_monitor_webtests` | ❌ | --- ## Test 232 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the details for my storage account +**Expected Tool:** `get_azure_load_testing_details` +**Prompt:** Get the load test with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592782 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.449229 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.434143 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.419250 | `get_azure_container_details` | ❌ | -| 5 | 0.419042 | `create_azure_storage` | ❌ | +| 1 | 0.612755 | `create_azure_load_testing` | ❌ | +| 2 | 0.592683 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 3 | 0.421766 | `update_azure_load_testing_configurations` | ❌ | +| 4 | 0.362418 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.349180 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 233 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the properties for blob in container in storage account +**Expected Tool:** `get_azure_load_testing_details` +**Prompt:** List all load testing resources in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.524375 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.521425 | `create_azure_storage` | ❌ | -| 3 | 0.457312 | `upload_azure_storage_blobs` | ❌ | -| 4 | 0.405959 | `get_azure_container_details` | ❌ | -| 5 | 0.348768 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.669717 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 2 | 0.609875 | `create_azure_load_testing` | ❌ | +| 3 | 0.493520 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.426767 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.421963 | `get_azure_capacity` | ❌ | --- ## Test 234 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the properties of the storage container in the storage account +**Expected Tool:** `create_azure_load_testing` +**Prompt:** Create a basic URL test using the following endpoint URL that runs for 30 minutes with 45 virtual users. The test name is with the test id and the load testing resource is in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543792 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.498174 | `create_azure_storage` | ❌ | -| 3 | 0.434495 | `get_azure_container_details` | ❌ | -| 4 | 0.398764 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.368710 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.542817 | `create_azure_load_testing` | ✅ **EXPECTED** | +| 2 | 0.491654 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.431906 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.425527 | `update_azure_load_testing_configurations` | ❌ | +| 5 | 0.409332 | `update_azure_monitor_webtests` | ❌ | --- ## Test 235 -**Expected Tool:** `get_azure_storage_details` -**Prompt:** Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings +**Expected Tool:** `create_azure_load_testing` +**Prompt:** Create a load test resource in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520985 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.476386 | `create_azure_storage` | ❌ | -| 3 | 0.430206 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.397596 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.397401 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.660181 | `create_azure_load_testing` | ✅ **EXPECTED** | +| 2 | 0.530657 | `get_azure_load_testing_details` | ❌ | +| 3 | 0.430087 | `create_azure_monitor_webtests` | ❌ | +| 4 | 0.411267 | `update_azure_load_testing_configurations` | ❌ | +| 5 | 0.374033 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 236 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create a new blob container named documents with container public access in storage account +**Expected Tool:** `create_azure_load_testing` +**Prompt:** Create a test run using the id for test in the load testing resource in resource group . Use the name of test run and description as ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546204 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.431996 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.379209 | `get_azure_storage_details` | ❌ | -| 4 | 0.318029 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.304649 | `search_microsoft_docs` | ❌ | +| 1 | 0.585612 | `create_azure_load_testing` | ✅ **EXPECTED** | +| 2 | 0.496772 | `update_azure_load_testing_configurations` | ❌ | +| 3 | 0.460907 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.420018 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.334626 | `update_azure_monitor_webtests` | ❌ | --- ## Test 237 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create a new storage account called testaccount123 in East US region +**Expected Tool:** `update_azure_load_testing_configurations` +**Prompt:** Update a test run display name as for the id for test in the load testing resource in resource group . ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.478014 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.354593 | `get_azure_storage_details` | ❌ | -| 3 | 0.329543 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.307994 | `create_azure_load_testing` | ❌ | -| 5 | 0.306614 | `get_azure_capacity` | ❌ | +| 1 | 0.577283 | `update_azure_load_testing_configurations` | ✅ **EXPECTED** | +| 2 | 0.501245 | `create_azure_load_testing` | ❌ | +| 3 | 0.443796 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.369079 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.360601 | `rename_azure_sql_databases` | ❌ | --- ## Test 238 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create a new storage account with Data Lake Storage Gen2 enabled +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Get details for AI Foundry resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.557678 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.441188 | `get_azure_storage_details` | ❌ | -| 3 | 0.432518 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.423712 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.395124 | `create_azure_workbooks` | ❌ | +| 1 | 0.599211 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.449921 | `get_azure_signalr_details` | ❌ | +| 3 | 0.425558 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.418851 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.418790 | `get_azure_messaging_service_details` | ❌ | --- ## Test 239 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create a storage account with premium performance and LRS replication +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Get the details of knowledge base in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.488344 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.437403 | `get_azure_storage_details` | ❌ | -| 3 | 0.406118 | `get_azure_capacity` | ❌ | -| 4 | 0.356649 | `create_azure_load_testing` | ❌ | -| 5 | 0.346863 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.659122 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.646614 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.486623 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.442045 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.441523 | `get_azure_signalr_details` | ❌ | --- ## Test 240 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create the container using blob public access in storage account +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Get the details of knowledge source in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.631937 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.487471 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.396535 | `get_azure_storage_details` | ❌ | -| 4 | 0.326507 | `get_azure_container_details` | ❌ | -| 5 | 0.316986 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.600909 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.585328 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.440329 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.437073 | `get_azure_signalr_details` | ❌ | +| 5 | 0.416989 | `get_azure_app_resource_details` | ❌ | --- ## Test 241 -**Expected Tool:** `create_azure_storage` -**Prompt:** Create the storage container mycontainer in storage account +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Get the schema configuration for knowledge index ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607096 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.450602 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.403164 | `get_azure_storage_details` | ❌ | -| 4 | 0.325431 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.308541 | `get_azure_container_details` | ❌ | +| 1 | 0.316895 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.310095 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.269536 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 4 | 0.268668 | `get_azure_best_practices` | ❌ | +| 5 | 0.262166 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 242 -**Expected Tool:** `upload_azure_storage_blobs` -**Prompt:** Upload file to storage blob in container in storage account +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all agents in my AI Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.623181 | `upload_azure_storage_blobs` | ✅ **EXPECTED** | -| 2 | 0.528682 | `create_azure_storage` | ❌ | -| 3 | 0.375458 | `get_azure_storage_details` | ❌ | -| 4 | 0.292612 | `deploy_azure_ai_models` | ❌ | -| 5 | 0.268633 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.521566 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.510202 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.502701 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 4 | 0.498204 | `connect_azure_ai_foundry_agents` | ❌ | +| 5 | 0.362174 | `use_azure_openai_models` | ❌ | --- ## Test 243 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all access policies in the Redis Cache +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all AI Foundry model deployments ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.598233 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.335003 | `get_azure_security_configurations` | ❌ | -| 3 | 0.304910 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.292789 | `get_azure_key_vault` | ❌ | -| 5 | 0.267608 | `get_azure_container_details` | ❌ | +| 1 | 0.562287 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.448496 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.440588 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.412586 | `connect_azure_ai_foundry_agents` | ❌ | +| 5 | 0.394938 | `execute_azure_deployments` | ❌ | --- ## Test 244 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all databases in the Redis Cluster +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all AI Foundry models ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.470782 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.389069 | `get_azure_databases_details` | ❌ | -| 3 | 0.387732 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.280568 | `get_azure_container_details` | ❌ | -| 5 | 0.254949 | `get_azure_security_configurations` | ❌ | +| 1 | 0.504981 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.403945 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.398748 | `use_azure_openai_models` | ❌ | +| 4 | 0.397053 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.367885 | `connect_azure_ai_foundry_agents` | ❌ | --- ## Test 245 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all Redis Caches in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all AI Foundry resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.580587 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.364783 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.342455 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.340686 | `get_azure_databases_details` | ❌ | -| 5 | 0.310757 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.579817 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.406256 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.405825 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.403903 | `connect_azure_ai_foundry_agents` | ❌ | +| 5 | 0.396690 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- ## Test 246 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all Redis Clusters in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all available OpenAI models in my Azure resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.567767 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.435562 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.414456 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.396583 | `get_azure_container_details` | ❌ | -| 5 | 0.383637 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.603264 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.498031 | `use_azure_openai_models` | ❌ | +| 3 | 0.484432 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.456947 | `get_azure_capacity` | ❌ | +| 5 | 0.435210 | `generate_azure_cli_commands` | ❌ | --- ## Test 247 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me my Redis Caches +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all Cognitive Search services in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520317 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.290258 | `get_azure_databases_details` | ❌ | -| 3 | 0.261894 | `get_azure_container_details` | ❌ | -| 4 | 0.252540 | `get_azure_key_vault` | ❌ | -| 5 | 0.252359 | `get_azure_security_configurations` | ❌ | +| 1 | 0.534733 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.453699 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.444086 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.419389 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.416112 | `get_azure_signalr_details` | ❌ | --- ## Test 248 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me my Redis Clusters +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all indexes in the Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498407 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.354127 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.342390 | `get_azure_container_details` | ❌ | -| 4 | 0.298268 | `get_azure_databases_details` | ❌ | -| 5 | 0.272676 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.482325 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.398786 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.364422 | `get_azure_signalr_details` | ❌ | +| 4 | 0.357199 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.356614 | `get_azure_security_configurations` | ❌ | --- ## Test 249 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the access policies in the Redis Cache +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all knowledge bases in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600085 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.322520 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.316812 | `get_azure_security_configurations` | ❌ | -| 4 | 0.305854 | `get_azure_key_vault` | ❌ | -| 5 | 0.305484 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.667963 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.621099 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.450304 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.444213 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.401759 | `get_azure_databases_details` | ❌ | --- ## Test 250 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the databases in the Redis Cluster +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all knowledge bases in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.456959 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.379510 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.372181 | `get_azure_databases_details` | ❌ | -| 4 | 0.280782 | `get_azure_container_details` | ❌ | -| 5 | 0.238852 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.531138 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.448783 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.337241 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.322260 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.320822 | `get_azure_container_details` | ❌ | --- ## Test 251 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the Redis Caches in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all knowledge indexes in my AI Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.553755 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.360815 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.335247 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.333569 | `get_azure_databases_details` | ❌ | -| 5 | 0.328519 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.515763 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.410647 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.404503 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.396550 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.366440 | `use_azure_openai_models` | ❌ | --- ## Test 252 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the Redis Clusters in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all knowledge sources in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.538790 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.424900 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.415817 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.400228 | `get_azure_container_details` | ❌ | -| 5 | 0.380760 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.654111 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.615349 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.446649 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.422491 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.416060 | `get_azure_signalr_details` | ❌ | --- ## Test 253 -**Expected Tool:** `browse_azure_marketplace_products` -**Prompt:** Get details about marketplace product +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** List all knowledge sources in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.424825 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | -| 2 | 0.376519 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.344490 | `get_azure_storage_details` | ❌ | -| 4 | 0.343786 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.302227 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.453376 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.433866 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.321039 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.300443 | `get_azure_container_details` | ❌ | +| 5 | 0.295001 | `get_azure_signalr_details` | ❌ | --- ## Test 254 -**Expected Tool:** `browse_azure_marketplace_products` -**Prompt:** Search for Microsoft products in the marketplace +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Search for instances of in the index in Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.712278 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | -| 2 | 0.464133 | `search_microsoft_docs` | ❌ | -| 3 | 0.387115 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.364792 | `deploy_resources_and_applications_to_azure` | ❌ | -| 5 | 0.344772 | `get_azure_databases_details` | ❌ | +| 1 | 0.507589 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.425742 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.309441 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.298713 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.291702 | `use_azure_openai_models` | ❌ | --- ## Test 255 -**Expected Tool:** `browse_azure_marketplace_products` -**Prompt:** Show me marketplace products from publisher +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me all AI Foundry model deployments ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.492651 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | -| 2 | 0.227398 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.217240 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.210581 | `get_azure_workbooks_details` | ❌ | -| 5 | 0.205515 | `get_azure_storage_details` | ❌ | +| 1 | 0.552427 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.462315 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.458855 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.435445 | `use_azure_openai_models` | ❌ | +| 5 | 0.425410 | `connect_azure_ai_foundry_agents` | ❌ | --- ## Test 256 -**Expected Tool:** `get_azure_capacity` -**Prompt:** Check usage information for in region +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me my Cognitive Search services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.484870 | `get_azure_capacity` | ✅ **EXPECTED** | -| 2 | 0.419610 | `get_azure_storage_details` | ❌ | -| 3 | 0.353830 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.350026 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.314300 | `get_azure_cache_for_redis_details` | ❌ | +| 1 | 0.492932 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.431878 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.414065 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.376911 | `get_azure_signalr_details` | ❌ | +| 5 | 0.364678 | `get_azure_resource_and_app_health_status` | ❌ | --- ## Test 257 -**Expected Tool:** `get_azure_capacity` -**Prompt:** Show me the available regions for these resource types +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the AI Foundry resources in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.398403 | `get_azure_capacity` | ✅ **EXPECTED** | -| 2 | 0.347081 | `get_azure_storage_details` | ❌ | -| 3 | 0.332100 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.313365 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.311621 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.588933 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.472651 | `connect_azure_ai_foundry_agents` | ❌ | +| 3 | 0.467078 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.463356 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.424608 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- ## Test 258 -**Expected Tool:** `get_azure_capacity` -**Prompt:** Tell me how many IP addresses I need for of +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the available agents in my AI Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.304686 | `get_azure_storage_details` | ❌ | -| 2 | 0.266002 | `get_azure_capacity` | ✅ **EXPECTED** | -| 3 | 0.225046 | `execute_azure_cli` | ❌ | -| 4 | 0.215121 | `edit_azure_databases` | ❌ | -| 5 | 0.212666 | `get_azure_container_details` | ❌ | +| 1 | 0.546302 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.534388 | `connect_azure_ai_foundry_agents` | ❌ | +| 3 | 0.530860 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.525204 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 5 | 0.421105 | `use_azure_openai_models` | ❌ | --- ## Test 259 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List all Event Grid subscriptions in subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the available AI Foundry models ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510172 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.431684 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.350456 | `get_azure_security_configurations` | ❌ | -| 4 | 0.326854 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.305112 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.522598 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.509261 | `use_azure_openai_models` | ❌ | +| 3 | 0.460083 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.459774 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.408332 | `connect_azure_ai_foundry_agents` | ❌ | --- ## Test 260 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List all Event Grid topics in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the Cognitive Search services in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.583585 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.433919 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.380839 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.375303 | `get_azure_security_configurations` | ❌ | -| 5 | 0.355380 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.543850 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.507116 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.468386 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.447907 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.425779 | `get_azure_app_resource_details` | ❌ | --- ## Test 261 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List all Event Grid topics in resource group in subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the details of the index in Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517003 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.485740 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.337256 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.332930 | `get_azure_security_configurations` | ❌ | -| 5 | 0.330989 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.494181 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.450784 | `get_azure_signalr_details` | ❌ | +| 3 | 0.422378 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.417792 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.406409 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 262 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List all Event Grid topics in subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the indexes in the Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.540770 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.397469 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.329801 | `get_azure_security_configurations` | ❌ | -| 4 | 0.325418 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.312790 | `get_azure_storage_details` | ❌ | +| 1 | 0.470270 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.407167 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.378470 | `get_azure_signalr_details` | ❌ | +| 4 | 0.372832 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.360971 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 263 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List Event Grid subscriptions for subscription in location +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge base in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508285 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.443596 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.351500 | `get_azure_security_configurations` | ❌ | -| 4 | 0.347938 | `get_azure_storage_details` | ❌ | -| 5 | 0.335511 | `get_azure_load_testing_details` | ❌ | +| 1 | 0.496619 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.435586 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.309789 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.302510 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.290298 | `get_azure_container_details` | ❌ | --- ## Test 264 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List Event Grid subscriptions for topic in resource group +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge bases in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.539106 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.499122 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.366263 | `get_azure_security_configurations` | ❌ | -| 4 | 0.341727 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.337568 | `get_azure_storage_details` | ❌ | +| 1 | 0.685811 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.620774 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.463222 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.439438 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.410728 | `get_azure_signalr_details` | ❌ | --- ## Test 265 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** List Event Grid subscriptions for topic in subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge bases in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.551835 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.422468 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.341614 | `get_azure_security_configurations` | ❌ | -| 4 | 0.321405 | `get_azure_storage_details` | ❌ | -| 5 | 0.319377 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.557843 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.456176 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.376595 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.337980 | `get_azure_container_details` | ❌ | +| 5 | 0.327510 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 266 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show all Event Grid subscriptions in my subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge indexes in my AI Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.553724 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.470453 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.429269 | `get_azure_security_configurations` | ❌ | -| 4 | 0.402480 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.358337 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.502676 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.463915 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.445706 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.423761 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 5 | 0.406387 | `use_azure_openai_models` | ❌ | --- ## Test 267 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show Event Grid subscriptions in resource group in subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge source in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.566739 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.506491 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.383948 | `get_azure_security_configurations` | ❌ | -| 4 | 0.348386 | `get_azure_storage_details` | ❌ | -| 5 | 0.346975 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.417775 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.373515 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.281319 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.261925 | `get_azure_container_details` | ❌ | +| 5 | 0.256054 | `get_azure_signalr_details` | ❌ | --- ## Test 268 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show me all Event Grid subscriptions for topic +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge sources in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552999 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.404707 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.344019 | `get_azure_security_configurations` | ❌ | -| 4 | 0.337543 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.317617 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.642192 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.627001 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.459537 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.420278 | `get_azure_signalr_details` | ❌ | +| 5 | 0.417829 | `get_azure_resource_and_app_health_status` | ❌ | --- ## Test 269 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show me the details of service bus queue +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the knowledge sources in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.602538 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.365590 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.364952 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.364635 | `get_azure_container_details` | ❌ | -| 5 | 0.354268 | `get_azure_storage_details` | ❌ | +| 1 | 0.465907 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.459306 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.362531 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.321876 | `get_azure_signalr_details` | ❌ | +| 5 | 0.307885 | `get_azure_container_details` | ❌ | --- ## Test 270 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show me the details of service bus subscription +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the OpenAI model deployments ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622635 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.380697 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.379021 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.357307 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.355318 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.442176 | `use_azure_openai_models` | ❌ | +| 2 | 0.414312 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.364747 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.351248 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.350036 | `execute_azure_deployments` | ❌ | --- ## Test 271 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show me the details of service bus topic +**Expected Tool:** `get_azure_ai_resources_details` +**Prompt:** Show me the schema for knowledge index in my AI Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.615458 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.363340 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.347442 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.343578 | `get_azure_container_details` | ❌ | -| 5 | 0.336113 | `get_azure_storage_details` | ❌ | +| 1 | 0.478034 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.412935 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.324853 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.324788 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.323317 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- ## Test 272 -**Expected Tool:** `get_azure_messaging_service_details` -**Prompt:** Show me the Event Grid topics in my subscription +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Ask knowledge base in search service to retrieve information about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.587666 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.444005 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.417523 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.364752 | `get_azure_security_configurations` | ❌ | -| 5 | 0.347933 | `get_azure_storage_details` | ❌ | +| 1 | 0.566491 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.464178 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.312268 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.305023 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.304985 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- ## Test 273 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** List all Data Explorer clusters in my subscription +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Find information about using knowledge base in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.589762 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.413814 | `get_azure_databases_details` | ❌ | -| 3 | 0.398499 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.385218 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.379204 | `get_azure_container_details` | ❌ | +| 1 | 0.564315 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.477702 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.330673 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.327179 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.321587 | `get_azure_messaging_service_details` | ❌ | --- ## Test 274 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** List all databases in the Data Explorer cluster +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Query knowledge base in search service about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546030 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.462094 | `get_azure_databases_details` | ❌ | -| 3 | 0.337758 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.295340 | `get_azure_container_details` | ❌ | -| 5 | 0.284549 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.516858 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.465631 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.353651 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.324671 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.306547 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 275 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** List all tables in the Data Explorer database in cluster +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Run a retrieval with knowledge base in Azure AI Search service for the query ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527003 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.410919 | `get_azure_databases_details` | ❌ | -| 3 | 0.305694 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.263811 | `get_azure_storage_details` | ❌ | -| 5 | 0.256147 | `get_azure_container_details` | ❌ | +| 1 | 0.669285 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.561779 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.450678 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.412118 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.390813 | `get_azure_data_explorer_kusto_details` | ❌ | --- ## Test 276 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me a data sample from the Data Explorer table in cluster +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Run a retrieval with knowledge base in search service for the query + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.524146 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.399606 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.322797 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.267991 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.264021 | `get_azure_data_explorer_kusto_details` | ❌ | + +--- + +## Test 277 + +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** Search knowledge base in Azure AI Search service for + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.691488 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.627605 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.462042 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.421014 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.415222 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 278 + +**Expected Tool:** `retrieve_azure_ai_knowledge_base_content` +**Prompt:** What does knowledge base in search service know about + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.530839 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.439695 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.308871 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.285910 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.282944 | `get_azure_data_explorer_kusto_details` | ❌ | + +--- + +## Test 279 + +**Expected Tool:** `use_azure_openai_models` +**Prompt:** Create a chat completion with the message "Hello, how are you today?" + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.389976 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 2 | 0.254256 | `send_azure_communication_messages` | ❌ | +| 3 | 0.205840 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.184631 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.150665 | `connect_azure_ai_foundry_agents` | ❌ | + +--- + +## Test 280 + +**Expected Tool:** `use_azure_openai_models` +**Prompt:** Create a completion with the prompt "What is Azure?" + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.517427 | `generate_azure_cli_commands` | ❌ | +| 2 | 0.472325 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 3 | 0.390310 | `execute_azure_deployments` | ❌ | +| 4 | 0.388807 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.377388 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 281 + +**Expected Tool:** `use_azure_openai_models` +**Prompt:** Create vector embeddings for my text using Azure OpenAI + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.682124 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 2 | 0.414187 | `recognize_speech_from_audio` | ❌ | +| 3 | 0.403186 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.401946 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.385329 | `get_azure_ai_resources_details` | ❌ | + +--- + +## Test 282 + +**Expected Tool:** `use_azure_openai_models` +**Prompt:** Generate embeddings for the text "Azure OpenAI Service" + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.620210 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 2 | 0.417187 | `recognize_speech_from_audio` | ❌ | +| 3 | 0.416775 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.388115 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 5 | 0.376550 | `get_azure_ai_resources_details` | ❌ | + +--- + +## Test 283 + +**Expected Tool:** `connect_azure_ai_foundry_agents` +**Prompt:** Query an agent in my AI foundry project + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.575213 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.514602 | `connect_azure_ai_foundry_agents` | ✅ **EXPECTED** | +| 3 | 0.510270 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.495795 | `get_azure_ai_resources_details` | ❌ | +| 5 | 0.385756 | `use_azure_openai_models` | ❌ | + +--- + +## Test 284 + +**Expected Tool:** `query_and_evaluate_azure_ai_foundry_agents` +**Prompt:** Query and evaluate an agent in my AI Foundry project for task_adherence + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.596620 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.590123 | `query_and_evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | +| 3 | 0.423635 | `connect_azure_ai_foundry_agents` | ❌ | +| 4 | 0.383588 | `get_azure_ai_resources_details` | ❌ | +| 5 | 0.319233 | `use_azure_openai_models` | ❌ | + +--- + +## Test 285 + +**Expected Tool:** `evaluate_azure_ai_foundry_agents` +**Prompt:** Evaluate the full query and response I got from my agent for task_adherence + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.426012 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.364688 | `evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | +| 3 | 0.262513 | `get_azure_resource_and_app_health_status` | ❌ | +| 4 | 0.249584 | `get_azure_ai_resources_details` | ❌ | +| 5 | 0.238878 | `get_azure_database_admin_configuration_details` | ❌ | + +--- + +## Test 286 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Get details about the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.623815 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.475469 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.430956 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.426398 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.422049 | `create_azure_storage` | ❌ | + +--- + +## Test 287 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Get the details about blob in the container in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.601381 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.478682 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.448552 | `get_azure_container_details` | ❌ | +| 4 | 0.427685 | `create_azure_storage` | ❌ | +| 5 | 0.415661 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 288 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List all blob containers in the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.510965 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.478409 | `create_azure_storage` | ❌ | +| 3 | 0.453205 | `upload_azure_storage_blobs` | ❌ | +| 4 | 0.357190 | `get_azure_container_details` | ❌ | +| 5 | 0.336886 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 289 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List all blobs in the blob container in the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.497314 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.463770 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.458536 | `create_azure_storage` | ❌ | +| 4 | 0.338651 | `get_azure_container_details` | ❌ | +| 5 | 0.309725 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 290 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List all storage accounts in my subscription including their location and SKU + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.611409 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.444366 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.441300 | `create_azure_storage` | ❌ | +| 4 | 0.428352 | `get_azure_capacity` | ❌ | +| 5 | 0.400577 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 291 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List the Azure Managed Lustre filesystems in my resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.583632 | `update_azure_managed_lustre_filesystems` | ❌ | +| 2 | 0.577262 | `create_azure_storage` | ❌ | +| 3 | 0.573528 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 4 | 0.432052 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.421279 | `get_azure_capacity` | ❌ | + +--- + +## Test 292 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List the Azure Managed Lustre filesystems in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.605946 | `update_azure_managed_lustre_filesystems` | ❌ | +| 2 | 0.596710 | `create_azure_storage` | ❌ | +| 3 | 0.593813 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 4 | 0.435301 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.431655 | `get_azure_subscriptions_and_resource_groups` | ❌ | + +--- + +## Test 293 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** List the Azure Managed Lustre SKUs available in location + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.597221 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.541443 | `update_azure_managed_lustre_filesystems` | ❌ | +| 3 | 0.531024 | `create_azure_storage` | ❌ | +| 4 | 0.463214 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.435044 | `get_azure_capacity` | ❌ | + +--- + +## Test 294 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me my storage accounts with whether hierarchical namespace (HNS) is enabled + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.493767 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.426243 | `create_azure_storage` | ❌ | +| 3 | 0.388186 | `get_azure_capacity` | ❌ | +| 4 | 0.384612 | `get_azure_security_configurations` | ❌ | +| 5 | 0.381052 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 295 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the blobs in the blob container in the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.501412 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.472639 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.454001 | `create_azure_storage` | ❌ | +| 4 | 0.397280 | `get_azure_container_details` | ❌ | +| 5 | 0.328169 | `get_azure_confidential_ledger_entries` | ❌ | + +--- + +## Test 296 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the containers in the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.520158 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.480540 | `create_azure_storage` | ❌ | +| 3 | 0.418173 | `upload_azure_storage_blobs` | ❌ | +| 4 | 0.408751 | `get_azure_container_details` | ❌ | +| 5 | 0.354495 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 297 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the details for my storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.591984 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.449230 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.419483 | `create_azure_storage` | ❌ | +| 4 | 0.419250 | `get_azure_container_details` | ❌ | +| 5 | 0.414316 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 298 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the properties for blob in container in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.546667 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.457238 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.439803 | `create_azure_storage` | ❌ | +| 4 | 0.405897 | `get_azure_container_details` | ❌ | +| 5 | 0.348646 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 299 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the properties of the storage container in the storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.559658 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.450881 | `create_azure_storage` | ❌ | +| 3 | 0.434495 | `get_azure_container_details` | ❌ | +| 4 | 0.398764 | `upload_azure_storage_blobs` | ❌ | +| 5 | 0.368711 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 300 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.512980 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.430206 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.425170 | `create_azure_storage` | ❌ | +| 4 | 0.397401 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.385193 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 301 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Tell me how many IP addresses I need for an Azure Managed Lustre filesystem of size using the SKU + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.584136 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.543122 | `create_azure_storage` | ❌ | +| 3 | 0.527934 | `update_azure_managed_lustre_filesystems` | ❌ | +| 4 | 0.421820 | `get_azure_capacity` | ❌ | +| 5 | 0.332218 | `generate_azure_cli_commands` | ❌ | + +--- + +## Test 302 + +**Expected Tool:** `get_azure_storage_details` +**Prompt:** Validate if the network can host Azure Managed Lustre filesystem of size using the SKU + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.574214 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.504246 | `update_azure_managed_lustre_filesystems` | ❌ | +| 3 | 0.472221 | `create_azure_storage` | ❌ | +| 4 | 0.393120 | `get_azure_capacity` | ❌ | +| 5 | 0.316611 | `publish_azure_eventgrid_events` | ❌ | + +--- + +## Test 303 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create a new blob container named documents with container public access in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.462833 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.431996 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.408422 | `get_azure_storage_details` | ❌ | +| 4 | 0.339283 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.318823 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 304 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create a new storage account called testaccount123 in East US region + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.460058 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.396664 | `get_azure_storage_details` | ❌ | +| 3 | 0.358873 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.328252 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.324048 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 305 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create a new storage account with Data Lake Storage Gen2 enabled + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.555692 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.477943 | `get_azure_storage_details` | ❌ | +| 3 | 0.469204 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.441689 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.436232 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 306 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create a storage account with premium performance and LRS replication + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.521592 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.483638 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.468735 | `get_azure_storage_details` | ❌ | +| 4 | 0.406117 | `get_azure_capacity` | ❌ | +| 5 | 0.356649 | `create_azure_load_testing` | ❌ | + +--- + +## Test 307 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create an Azure Managed Lustre filesystem with name , size , SKU , and subnet for availability zone in location . Maintenance should occur on at + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.640777 | `update_azure_managed_lustre_filesystems` | ❌ | +| 2 | 0.626919 | `create_azure_storage` | ✅ **EXPECTED** | +| 3 | 0.533351 | `get_azure_storage_details` | ❌ | +| 4 | 0.363841 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.339901 | `get_azure_capacity` | ❌ | + +--- + +## Test 308 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create the container using blob public access in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.532843 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.487514 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.440567 | `get_azure_storage_details` | ❌ | +| 4 | 0.345015 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.326599 | `get_azure_container_details` | ❌ | + +--- + +## Test 309 + +**Expected Tool:** `create_azure_storage` +**Prompt:** Create the storage container mycontainer in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.534269 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.450592 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.430175 | `get_azure_storage_details` | ❌ | +| 4 | 0.356380 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.328880 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 310 + +**Expected Tool:** `update_azure_managed_lustre_filesystems` +**Prompt:** Update the maintenance window of the Azure Managed Lustre filesystem to at + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.664565 | `update_azure_managed_lustre_filesystems` | ✅ **EXPECTED** | +| 2 | 0.449482 | `create_azure_storage` | ❌ | +| 3 | 0.368632 | `get_azure_storage_details` | ❌ | +| 4 | 0.327263 | `edit_azure_databases` | ❌ | +| 5 | 0.323886 | `edit_azure_workbooks` | ❌ | + +--- + +## Test 311 + +**Expected Tool:** `upload_azure_storage_blobs` +**Prompt:** Upload file to storage blob in container in storage account + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.623181 | `upload_azure_storage_blobs` | ✅ **EXPECTED** | +| 2 | 0.466442 | `create_azure_storage` | ❌ | +| 3 | 0.419370 | `get_azure_storage_details` | ❌ | +| 4 | 0.308477 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.268633 | `import_azure_key_vault_certificates` | ❌ | + +--- + +## Test 312 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** List all access policies in the Redis Cache + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.598205 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.335003 | `get_azure_security_configurations` | ❌ | +| 3 | 0.304910 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.290564 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.267608 | `get_azure_container_details` | ❌ | + +--- + +## Test 313 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** List all databases in the Redis Cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.470698 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.389069 | `get_azure_databases_details` | ❌ | +| 3 | 0.387732 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.296636 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.280567 | `get_azure_container_details` | ❌ | + +--- + +## Test 314 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** List all Redis Caches in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.580574 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.364804 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.340629 | `get_azure_databases_details` | ❌ | +| 4 | 0.321291 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.310751 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 315 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** List all Redis Clusters in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.567687 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.435563 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.414456 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.396583 | `get_azure_container_details` | ❌ | +| 5 | 0.364785 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 316 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me my Redis Caches + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.520310 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.290240 | `get_azure_databases_details` | ❌ | +| 3 | 0.261870 | `get_azure_container_details` | ❌ | +| 4 | 0.252356 | `get_azure_security_configurations` | ❌ | +| 5 | 0.246602 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 317 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me my Redis Clusters + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.498314 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.354127 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.342390 | `get_azure_container_details` | ❌ | +| 4 | 0.298268 | `get_azure_databases_details` | ❌ | +| 5 | 0.272676 | `get_azure_database_admin_configuration_details` | ❌ | + +--- + +## Test 318 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me the access policies in the Redis Cache + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.600094 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.322512 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.316775 | `get_azure_security_configurations` | ❌ | +| 4 | 0.309470 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.305487 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 319 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me the databases in the Redis Cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.456878 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.379510 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.372182 | `get_azure_databases_details` | ❌ | +| 4 | 0.280782 | `get_azure_container_details` | ❌ | +| 5 | 0.277615 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 320 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me the Redis Caches in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.553755 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.360815 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.335247 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.333570 | `get_azure_databases_details` | ❌ | +| 5 | 0.308733 | `get_azure_container_details` | ❌ | + +--- + +## Test 321 + +**Expected Tool:** `get_azure_cache_for_redis_details` +**Prompt:** Show me the Redis Clusters in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.538723 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | +| 2 | 0.424900 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.415817 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.400228 | `get_azure_container_details` | ❌ | +| 5 | 0.364169 | `get_azure_databases_details` | ❌ | + +--- + +## Test 322 + +**Expected Tool:** `browse_azure_marketplace_products` +**Prompt:** Get details about marketplace product + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.424825 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | +| 2 | 0.376519 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.359701 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.324404 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.310262 | `get_azure_storage_details` | ❌ | + +--- + +## Test 323 + +**Expected Tool:** `browse_azure_marketplace_products` +**Prompt:** Search for Microsoft products in the marketplace + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.712278 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | +| 2 | 0.379892 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.364792 | `deploy_azure_resources_and_applications` | ❌ | +| 4 | 0.353621 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 5 | 0.344772 | `get_azure_databases_details` | ❌ | + +--- + +## Test 324 + +**Expected Tool:** `browse_azure_marketplace_products` +**Prompt:** Show me marketplace products from publisher + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.492651 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | +| 2 | 0.225495 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.218384 | `get_azure_ai_resources_details` | ❌ | +| 4 | 0.215330 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 5 | 0.210581 | `get_azure_workbooks_details` | ❌ | + +--- + +## Test 325 + +**Expected Tool:** `get_azure_capacity` +**Prompt:** Check usage information for in region + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.484871 | `get_azure_capacity` | ✅ **EXPECTED** | +| 2 | 0.397452 | `get_azure_storage_details` | ❌ | +| 3 | 0.353830 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.350140 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.350026 | `get_azure_virtual_desktop_details` | ❌ | + +--- + +## Test 326 + +**Expected Tool:** `get_azure_capacity` +**Prompt:** Show me the available regions for these resource types + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.398404 | `get_azure_capacity` | ✅ **EXPECTED** | +| 2 | 0.332100 | `get_azure_load_testing_details` | ❌ | +| 3 | 0.326530 | `get_azure_storage_details` | ❌ | +| 4 | 0.322856 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.313365 | `get_azure_subscriptions_and_resource_groups` | ❌ | + +--- + +## Test 327 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Get the details of my consumer group in my event hub , namespace , and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.549915 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.537248 | `edit_azure_data_analytics_resources` | ❌ | +| 3 | 0.417076 | `get_azure_signalr_details` | ❌ | +| 4 | 0.413392 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.405636 | `get_azure_subscriptions_and_resource_groups` | ❌ | + +--- + +## Test 328 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Get the details of my event hub in my namespace and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.554775 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.510810 | `edit_azure_data_analytics_resources` | ❌ | +| 3 | 0.453455 | `get_azure_signalr_details` | ❌ | +| 4 | 0.423322 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.415404 | `get_azure_app_resource_details` | ❌ | + +--- + +## Test 329 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Get the details of my namespace in my resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.508903 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.468746 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.451446 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.432774 | `get_azure_signalr_details` | ❌ | +| 5 | 0.429765 | `get_azure_storage_details` | ❌ | + +--- + +## Test 330 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all consumer groups in my event hub in namespace + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.552301 | `edit_azure_data_analytics_resources` | ❌ | +| 2 | 0.469857 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.379261 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.360262 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.338916 | `get_azure_signalr_details` | ❌ | + +--- + +## Test 331 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Grid subscriptions in subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.445063 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.431684 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.414496 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.350456 | `get_azure_security_configurations` | ❌ | +| 5 | 0.335742 | `edit_azure_data_analytics_resources` | ❌ | + +--- + +## Test 332 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Grid topics in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.514030 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.510686 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.433919 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.380839 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.375303 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 333 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Grid topics in resource group in subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.517196 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.473028 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.440138 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.367530 | `edit_azure_data_analytics_resources` | ❌ | +| 5 | 0.337400 | `get_azure_load_testing_details` | ❌ | + +--- + +## Test 334 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Grid topics in subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.484028 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.473797 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.397469 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.335248 | `edit_azure_data_analytics_resources` | ❌ | +| 5 | 0.329802 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 335 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Hubs in my namespace + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.538223 | `edit_azure_data_analytics_resources` | ❌ | +| 2 | 0.489438 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.381085 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.361397 | `get_azure_signalr_details` | ❌ | +| 5 | 0.332270 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 336 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List all Event Hubs namespaces in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.517141 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.508146 | `edit_azure_data_analytics_resources` | ❌ | +| 3 | 0.415741 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.373001 | `get_azure_security_configurations` | ❌ | +| 5 | 0.366739 | `publish_azure_eventgrid_events` | ❌ | + +--- + +## Test 337 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List Event Grid subscriptions for subscription in location + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.443596 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.439593 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.413135 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.352198 | `get_azure_storage_details` | ❌ | +| 5 | 0.351500 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 338 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List Event Grid subscriptions for topic in resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.499123 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.494205 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.473537 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.366263 | `get_azure_security_configurations` | ❌ | +| 5 | 0.347265 | `edit_azure_data_analytics_resources` | ❌ | + +--- + +## Test 339 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** List Event Grid subscriptions for topic in subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.472417 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.469761 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.422468 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.341614 | `get_azure_security_configurations` | ❌ | +| 5 | 0.319377 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 340 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show all Event Grid subscriptions in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.480398 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.470532 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.453705 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.429371 | `get_azure_security_configurations` | ❌ | +| 5 | 0.402515 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 341 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show Event Grid subscriptions in resource group in subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.566739 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.483011 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.406254 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.383949 | `get_azure_security_configurations` | ❌ | +| 5 | 0.375086 | `edit_azure_data_analytics_resources` | ❌ | + +--- + +## Test 342 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show me all Event Grid subscriptions for topic + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.498244 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.475101 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.404707 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.344019 | `get_azure_security_configurations` | ❌ | +| 5 | 0.337543 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 343 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show me the details of service bus queue + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.577545 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.422716 | `get_azure_signalr_details` | ❌ | +| 3 | 0.384758 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.365590 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.364952 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 344 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show me the details of service bus subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.565484 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.432405 | `get_azure_signalr_details` | ❌ | +| 3 | 0.397741 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.380696 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.379021 | `get_azure_app_config_settings` | ❌ | + +--- + +## Test 345 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show me the details of service bus topic + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.568951 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.408956 | `get_azure_signalr_details` | ❌ | +| 3 | 0.375835 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.363340 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.347442 | `get_azure_virtual_desktop_details` | ❌ | + +--- + +## Test 346 + +**Expected Tool:** `get_azure_messaging_service_details` +**Prompt:** Show me the Event Grid topics in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.528995 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.519422 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.444005 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.417523 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.364752 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 347 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Create a new consumer group in my event hub , namespace , and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.567860 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.401880 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.380930 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.343917 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.334668 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 348 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Create a new event hub in my namespace and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.537246 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.420610 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.400466 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.365714 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.364259 | `create_azure_key_vault_items` | ❌ | + +--- + +## Test 349 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Create an new namespace in my resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.381243 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.380652 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.352027 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.329775 | `create_azure_load_testing` | ❌ | +| 5 | 0.324124 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 350 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Delete my consumer group in my event hub , namespace , and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.669651 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.382770 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.330235 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.311527 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.305503 | `edit_azure_workbooks` | ❌ | + +--- + +## Test 351 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Delete my event hub in my namespace and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.658804 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.371664 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.359810 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.356070 | `edit_azure_app_config_settings` | ❌ | +| 5 | 0.340097 | `edit_azure_workbooks` | ❌ | + +--- + +## Test 352 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Delete my namespace in my resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.508241 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.364331 | `edit_azure_app_config_settings` | ❌ | +| 3 | 0.328435 | `edit_azure_workbooks` | ❌ | +| 4 | 0.325612 | `delete_azure_database_admin_configurations` | ❌ | +| 5 | 0.323482 | `edit_azure_sql_databases_and_servers` | ❌ | + +--- + +## Test 353 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Update my consumer group in my event hub , namespace , and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.738618 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.439270 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.386803 | `update_azure_managed_lustre_filesystems` | ❌ | +| 4 | 0.373932 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.362897 | `edit_azure_workbooks` | ❌ | + +--- + +## Test 354 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Update my event hub in my namespace and resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.708436 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.440297 | `update_azure_managed_lustre_filesystems` | ❌ | +| 3 | 0.431338 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.423791 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.389301 | `edit_azure_workbooks` | ❌ | + +--- + +## Test 355 + +**Expected Tool:** `edit_azure_data_analytics_resources` +**Prompt:** Update my namespace in my resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.538963 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.416006 | `update_azure_managed_lustre_filesystems` | ❌ | +| 3 | 0.368404 | `edit_azure_workbooks` | ❌ | +| 4 | 0.368253 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.352082 | `edit_azure_sql_databases_and_servers` | ❌ | + +--- + +## Test 356 + +**Expected Tool:** `publish_azure_eventgrid_events` +**Prompt:** Publish an event to Event Grid topic using with the following data + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.732200 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.364422 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.327560 | `edit_azure_data_analytics_resources` | ❌ | +| 4 | 0.289079 | `get_azure_best_practices` | ❌ | +| 5 | 0.286527 | `send_azure_communication_messages` | ❌ | + +--- + +## Test 357 + +**Expected Tool:** `publish_azure_eventgrid_events` +**Prompt:** Publish event to my Event Grid topic with the following events + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.647380 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.381287 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.352434 | `edit_azure_data_analytics_resources` | ❌ | +| 4 | 0.297256 | `send_azure_communication_messages` | ❌ | +| 5 | 0.294962 | `upload_azure_storage_blobs` | ❌ | + +--- + +## Test 358 + +**Expected Tool:** `publish_azure_eventgrid_events` +**Prompt:** Send an event to Event Grid topic in resource group with + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.594047 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.386591 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.369923 | `edit_azure_data_analytics_resources` | ❌ | +| 4 | 0.295586 | `send_azure_communication_messages` | ❌ | +| 5 | 0.289053 | `get_azure_subscriptions_and_resource_groups` | ❌ | + +--- + +## Test 359 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** List all Data Explorer clusters in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.589762 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.413814 | `get_azure_databases_details` | ❌ | +| 3 | 0.398499 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.385167 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.379203 | `get_azure_container_details` | ❌ | + +--- + +## Test 360 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** List all databases in the Data Explorer cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.546030 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.462094 | `get_azure_databases_details` | ❌ | +| 3 | 0.337694 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.304208 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.295340 | `get_azure_container_details` | ❌ | + +--- + +## Test 361 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** List all tables in the Data Explorer database in cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.526929 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.410927 | `get_azure_databases_details` | ❌ | +| 3 | 0.305623 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.256124 | `get_azure_container_details` | ❌ | +| 5 | 0.244911 | `get_azure_virtual_desktop_details` | ❌ | + +--- + +## Test 362 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me a data sample from the Data Explorer table in cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.512442 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.313016 | `get_azure_databases_details` | ❌ | +| 3 | 0.263223 | `get_azure_confidential_ledger_entries` | ❌ | +| 4 | 0.248948 | `get_azure_container_details` | ❌ | +| 5 | 0.242279 | `get_azure_cache_for_redis_details` | ❌ | + +--- + +## Test 363 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me all items that contain the word in the Data Explorer table in cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.428985 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.359004 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.305757 | `get_azure_databases_details` | ❌ | +| 4 | 0.297599 | `get_azure_ai_resources_details` | ❌ | +| 5 | 0.264111 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 364 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me my Data Explorer clusters + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.533350 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.337173 | `get_azure_databases_details` | ❌ | +| 3 | 0.337078 | `get_azure_container_details` | ❌ | +| 4 | 0.315555 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.305856 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 365 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me the Data Explorer clusters in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.584974 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.420030 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.415732 | `get_azure_databases_details` | ❌ | +| 4 | 0.404725 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.400086 | `get_azure_container_details` | ❌ | + +--- + +## Test 366 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me the databases in the Data Explorer cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.535152 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.443460 | `get_azure_databases_details` | ❌ | +| 3 | 0.327990 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.301627 | `get_azure_container_details` | ❌ | +| 5 | 0.295149 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 367 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me the details of the Data Explorer cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.603734 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.416961 | `get_azure_container_details` | ❌ | +| 3 | 0.382586 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.365816 | `get_azure_databases_details` | ❌ | +| 5 | 0.363158 | `get_azure_signalr_details` | ❌ | + +--- + +## Test 368 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me the schema for table in the Data Explorer database in cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.475347 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.367386 | `get_azure_databases_details` | ❌ | +| 3 | 0.251481 | `get_azure_best_practices` | ❌ | +| 4 | 0.242454 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.242382 | `design_azure_architecture` | ❌ | + +--- + +## Test 369 + +**Expected Tool:** `get_azure_data_explorer_kusto_details` +**Prompt:** Show me the tables in the Data Explorer database in cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.521279 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.401926 | `get_azure_databases_details` | ❌ | +| 3 | 0.301626 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.271168 | `get_azure_container_details` | ❌ | +| 5 | 0.258457 | `get_azure_database_admin_configuration_details` | ❌ | + +--- + +## Test 370 + +**Expected Tool:** `create_azure_database_admin_configurations` +**Prompt:** Add a firewall rule to allow access from IP range to for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.619667 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.497535 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.339582 | `edit_azure_databases` | ❌ | +| 4 | 0.339238 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.310935 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 371 + +**Expected Tool:** `create_azure_database_admin_configurations` +**Prompt:** Create a firewall rule for my Azure SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.769949 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.659650 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.476833 | `edit_azure_databases` | ❌ | +| 4 | 0.455132 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.452335 | `create_azure_sql_databases_and_servers` | ❌ | + +--- + +## Test 372 + +**Expected Tool:** `create_azure_database_admin_configurations` +**Prompt:** Create a new firewall rule named for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.670172 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.546667 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.409535 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.399140 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.370061 | `get_azure_database_admin_configuration_details` | ❌ | + +--- + +## Test 373 + +**Expected Tool:** `delete_azure_database_admin_configurations` +**Prompt:** Delete a firewall rule from my Azure SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.725925 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.684225 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.507485 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.446832 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.434498 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 374 + +**Expected Tool:** `delete_azure_database_admin_configurations` +**Prompt:** Delete firewall rule for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.691123 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.657272 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.411100 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.410580 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.398776 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 375 + +**Expected Tool:** `delete_azure_database_admin_configurations` +**Prompt:** Remove the firewall rule from SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.662278 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.610044 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.374118 | `edit_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.368243 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.351139 | `rename_azure_sql_databases` | ❌ | + +--- + +## Test 376 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** List all elastic pools in SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.550794 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.437477 | `get_azure_databases_details` | ❌ | +| 3 | 0.418188 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.411871 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.370734 | `delete_azure_database_admin_configurations` | ❌ | + +--- + +## Test 377 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** List all firewall rules for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.659544 | `create_azure_database_admin_configurations` | ❌ | +| 2 | 0.635949 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.509163 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 4 | 0.344890 | `get_azure_security_configurations` | ❌ | +| 5 | 0.332632 | `get_azure_databases_details` | ❌ | + +--- + +## Test 378 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** List Microsoft Entra ID administrators for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.498356 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.362041 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.358939 | `get_azure_security_configurations` | ❌ | +| 4 | 0.343594 | `get_azure_databases_details` | ❌ | +| 5 | 0.334656 | `delete_azure_database_admin_configurations` | ❌ | + +--- + +## Test 379 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** Show me the elastic pools configured for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.602459 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.445269 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.424559 | `get_azure_databases_details` | ❌ | +| 4 | 0.414641 | `edit_azure_databases` | ❌ | +| 5 | 0.400359 | `get_azure_container_details` | ❌ | + +--- + +## Test 380 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** Show me the Entra ID administrators configured for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.498316 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.325040 | `create_azure_database_admin_configurations` | ❌ | +| 3 | 0.300436 | `add_azure_app_service_database` | ❌ | +| 4 | 0.299005 | `get_azure_databases_details` | ❌ | +| 5 | 0.294052 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 381 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** Show me the firewall rules for SQL server + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.659102 | `create_azure_database_admin_configurations` | ❌ | +| 2 | 0.611917 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.486395 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 4 | 0.361115 | `edit_azure_databases` | ❌ | +| 5 | 0.322908 | `get_azure_security_configurations` | ❌ | + +--- + +## Test 382 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** What elastic pools are available in my SQL server ? + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.515299 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.442723 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.412957 | `edit_azure_databases` | ❌ | +| 4 | 0.411657 | `get_azure_databases_details` | ❌ | +| 5 | 0.380417 | `get_azure_capacity` | ❌ | + +--- + +## Test 383 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** What firewall rules are configured for my SQL server ? + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.657075 | `create_azure_database_admin_configurations` | ❌ | +| 2 | 0.595199 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.493189 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 4 | 0.358803 | `edit_azure_databases` | ❌ | +| 5 | 0.329180 | `edit_azure_sql_databases_and_servers` | ❌ | + +--- + +## Test 384 + +**Expected Tool:** `get_azure_database_admin_configuration_details` +**Prompt:** What Microsoft Entra ID administrators are set up for my SQL server ? + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.451827 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.340144 | `add_azure_app_service_database` | ❌ | +| 3 | 0.332608 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.331531 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.331515 | `create_azure_sql_databases_and_servers` | ❌ | + +--- + +## Test 385 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Get details for nodepool in AKS cluster in + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.591315 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.489551 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.461100 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.438929 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.434421 | `get_azure_signalr_details` | ❌ | + +--- + +## Test 386 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Get the configuration of AKS cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.525642 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.515594 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.423935 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.384930 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.379742 | `get_azure_cache_for_redis_details` | ❌ | + +--- + +## Test 387 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List all AKS clusters in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.541013 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.472911 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.459564 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.420414 | `get_azure_security_configurations` | ❌ | +| 5 | 0.403360 | `get_azure_virtual_desktop_details` | ❌ | + +--- + +## Test 388 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List all Azure Container Registries in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.585525 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.460024 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.428348 | `get_azure_storage_details` | ❌ | +| 4 | 0.427150 | `get_azure_security_configurations` | ❌ | +| 5 | 0.410049 | `browse_azure_marketplace_products` | ❌ | + +--- + +## Test 389 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List all container registry repositories in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.514903 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.394679 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.385192 | `get_azure_storage_details` | ❌ | +| 4 | 0.351749 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.349969 | `get_azure_cache_for_redis_details` | ❌ | + +--- + +## Test 390 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List container registries in resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.489483 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.382508 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.365207 | `get_azure_storage_details` | ❌ | +| 4 | 0.356837 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.349921 | `get_azure_load_testing_details` | ❌ | + +--- + +## Test 391 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List nodepools for AKS cluster in + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.542296 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.417414 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.385553 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.372743 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.371410 | `get_azure_data_explorer_kusto_details` | ❌ | + +--- + +## Test 392 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** List repositories in the container registry + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.452303 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.285101 | `get_azure_cache_for_redis_details` | ❌ | +| 3 | 0.277147 | `get_azure_storage_details` | ❌ | +| 4 | 0.265399 | `get_azure_security_configurations` | ❌ | +| 5 | 0.261745 | `create_azure_storage` | ❌ | + +--- + +## Test 393 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me my Azure Container Registries + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.593337 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.419226 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.395307 | `get_azure_storage_details` | ❌ | +| 4 | 0.390515 | `get_azure_security_configurations` | ❌ | +| 5 | 0.384917 | `get_azure_cache_for_redis_details` | ❌ | + +--- + +## Test 394 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me my Azure Kubernetes Service clusters + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.536299 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.472664 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.416305 | `get_azure_signalr_details` | ❌ | +| 4 | 0.415764 | `get_azure_security_configurations` | ❌ | +| 5 | 0.400655 | `get_azure_subscriptions_and_resource_groups` | ❌ | + +--- + +## Test 395 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me my container registry repositories + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.485764 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.310577 | `get_azure_security_configurations` | ❌ | +| 3 | 0.310265 | `get_azure_storage_details` | ❌ | +| 4 | 0.294726 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.293685 | `get_azure_cache_for_redis_details` | ❌ | + +--- + +## Test 396 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the configuration for nodepool in AKS cluster in resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.516160 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.442585 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.410309 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.394415 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.360302 | `get_azure_signalr_details` | ❌ | + +--- + +## Test 397 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the container registries in my subscription + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.547872 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.432956 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.391672 | `get_azure_storage_details` | ❌ | +| 4 | 0.385551 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.361321 | `get_azure_messaging_service_details` | ❌ | + +--- + +## Test 398 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the container registries in resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.510429 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.431510 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.355156 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.353883 | `get_azure_storage_details` | ❌ | +| 5 | 0.343124 | `get_azure_load_testing_details` | ❌ | + +--- + +## Test 399 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the details of AKS cluster in resource group + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.579908 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.459131 | `get_azure_signalr_details` | ❌ | +| 3 | 0.444584 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.424458 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.424320 | `get_azure_virtual_desktop_details` | ❌ | + +--- + +## Test 400 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the network configuration for AKS cluster + +### Results + +| Rank | Score | Tool | Status | +|------|-------|------|--------| +| 1 | 0.452245 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.375014 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.357934 | `get_azure_signalr_details` | ❌ | +| 4 | 0.343230 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.322237 | `get_azure_data_explorer_kusto_details` | ❌ | + +--- + +## Test 401 + +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the nodepool list for AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512442 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.313016 | `get_azure_databases_details` | ❌ | -| 3 | 0.248948 | `get_azure_container_details` | ❌ | -| 4 | 0.242332 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.229602 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.542863 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.412997 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.387109 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.378755 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.373530 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 277 +## Test 402 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me all items that contain the word in the Data Explorer table in cluster +**Expected Tool:** `get_azure_container_details` +**Prompt:** Show me the repositories in the container registry ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.428985 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.305757 | `get_azure_databases_details` | ❌ | -| 3 | 0.305238 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.264111 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.241148 | `get_azure_container_details` | ❌ | +| 1 | 0.463224 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.287737 | `get_azure_cache_for_redis_details` | ❌ | +| 3 | 0.277348 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.273850 | `get_azure_storage_details` | ❌ | +| 5 | 0.267446 | `get_azure_key_vault_items` | ❌ | --- -## Test 278 +## Test 403 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me my Data Explorer clusters +**Expected Tool:** `get_azure_container_details` +**Prompt:** What AKS clusters do I have? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533350 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.337173 | `get_azure_databases_details` | ❌ | -| 3 | 0.337078 | `get_azure_container_details` | ❌ | -| 4 | 0.315634 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.313020 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.580085 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.423943 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.351001 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.349750 | `get_azure_storage_details` | ❌ | +| 5 | 0.349339 | `get_azure_signalr_details` | ❌ | --- -## Test 279 +## Test 404 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me the Data Explorer clusters in my subscription +**Expected Tool:** `get_azure_container_details` +**Prompt:** What are the details of my AKS cluster in ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.584941 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.420001 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.415745 | `get_azure_databases_details` | ❌ | -| 4 | 0.404684 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.400035 | `get_azure_container_details` | ❌ | +| 1 | 0.607448 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.511645 | `get_azure_signalr_details` | ❌ | +| 3 | 0.458216 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.457991 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.451232 | `get_azure_storage_details` | ❌ | --- -## Test 280 +## Test 405 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me the databases in the Data Explorer cluster +**Expected Tool:** `get_azure_container_details` +**Prompt:** What is the setup of nodepool for AKS cluster in ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.535152 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.443460 | `get_azure_databases_details` | ❌ | -| 3 | 0.328037 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.301627 | `get_azure_container_details` | ❌ | -| 5 | 0.284454 | `get_azure_storage_details` | ❌ | +| 1 | 0.489130 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.347326 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.338797 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.317925 | `get_azure_signalr_details` | ❌ | +| 5 | 0.315321 | `generate_azure_cli_commands` | ❌ | --- -## Test 281 +## Test 406 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me the details of the Data Explorer cluster +**Expected Tool:** `get_azure_container_details` +**Prompt:** What nodepools do I have for AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.603734 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.416961 | `get_azure_container_details` | ❌ | -| 3 | 0.382673 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.365816 | `get_azure_databases_details` | ❌ | -| 5 | 0.359112 | `get_azure_storage_details` | ❌ | +| 1 | 0.532501 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.389162 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.362346 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.358739 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.355086 | `get_azure_signalr_details` | ❌ | --- -## Test 282 +## Test 407 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me the schema for table in the Data Explorer database in cluster +**Expected Tool:** `get_azure_virtual_desktop_details` +**Prompt:** List all host pools in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.475232 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.367299 | `get_azure_databases_details` | ❌ | -| 3 | 0.251410 | `get_azure_best_practices` | ❌ | -| 4 | 0.242322 | `design_azure_architecture` | ❌ | -| 5 | 0.241156 | `get_azure_cache_for_redis_details` | ❌ | +| 1 | 0.550232 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | +| 2 | 0.453703 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.442934 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.412970 | `get_azure_container_details` | ❌ | +| 5 | 0.396066 | `get_azure_capacity` | ❌ | --- -## Test 283 +## Test 408 -**Expected Tool:** `get_azure_data_explorer_kusto_details` -**Prompt:** Show me the tables in the Data Explorer database in cluster +**Expected Tool:** `get_azure_virtual_desktop_details` +**Prompt:** List all session hosts in host pool ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521279 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.401926 | `get_azure_databases_details` | ❌ | -| 3 | 0.301709 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.271168 | `get_azure_container_details` | ❌ | -| 5 | 0.266107 | `get_azure_storage_details` | ❌ | +| 1 | 0.607532 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | +| 2 | 0.364628 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.319120 | `get_azure_security_configurations` | ❌ | +| 4 | 0.312479 | `get_azure_signalr_details` | ❌ | +| 5 | 0.307420 | `get_azure_container_details` | ❌ | --- -## Test 284 +## Test 409 -**Expected Tool:** `create_azure_database_admin_configurations` -**Prompt:** Add a firewall rule to allow access from IP range to for SQL server +**Expected Tool:** `get_azure_virtual_desktop_details` +**Prompt:** List all user sessions on session host in host pool ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619667 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.497535 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.339582 | `edit_azure_databases` | ❌ | -| 4 | 0.339238 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.210520 | `get_azure_databases_details` | ❌ | +| 1 | 0.611133 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | +| 2 | 0.335733 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.313084 | `get_azure_security_configurations` | ❌ | +| 4 | 0.283932 | `get_azure_signalr_details` | ❌ | +| 5 | 0.265858 | `get_azure_container_details` | ❌ | --- -## Test 285 +## Test 410 -**Expected Tool:** `create_azure_database_admin_configurations` -**Prompt:** Create a firewall rule for my Azure SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** Describe the SignalR runtime in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.769894 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.659595 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.476818 | `edit_azure_databases` | ❌ | -| 4 | 0.455060 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.349711 | `get_azure_databases_details` | ❌ | +| 1 | 0.711335 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.438870 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.387114 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.370390 | `edit_azure_data_analytics_resources` | ❌ | +| 5 | 0.363831 | `get_azure_capacity` | ❌ | --- -## Test 286 +## Test 411 -**Expected Tool:** `create_azure_database_admin_configurations` -**Prompt:** Create a new firewall rule named for SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** Get information about my SignalR runtime in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.670172 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.546667 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.370061 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.333972 | `edit_azure_databases` | ❌ | -| 5 | 0.250896 | `create_azure_workbooks` | ❌ | +| 1 | 0.729127 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.467032 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.452790 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.447727 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.433126 | `get_azure_app_config_settings` | ❌ | --- -## Test 287 +## Test 412 -**Expected Tool:** `delete_azure_database_admin_configurations` -**Prompt:** Delete a firewall rule from my Azure SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** List all SignalRs in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.725926 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.684224 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.446833 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.433064 | `edit_azure_databases` | ❌ | -| 5 | 0.365336 | `edit_azure_workbooks` | ❌ | +| 1 | 0.497376 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.390874 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.360482 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.336211 | `get_azure_security_configurations` | ❌ | +| 5 | 0.314121 | `browse_azure_marketplace_products` | ❌ | --- -## Test 288 +## Test 413 -**Expected Tool:** `delete_azure_database_admin_configurations` -**Prompt:** Delete firewall rule for SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** Show all the SignalRs information in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.691123 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.657273 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.410580 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.364151 | `edit_azure_databases` | ❌ | -| 5 | 0.287813 | `get_azure_security_configurations` | ❌ | +| 1 | 0.641102 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.499456 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.461902 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.420390 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.410032 | `get_azure_capacity` | ❌ | --- -## Test 289 +## Test 414 -**Expected Tool:** `delete_azure_database_admin_configurations` -**Prompt:** Remove the firewall rule from SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** Show me the details of SignalR ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.662278 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.610044 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.368243 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.299807 | `edit_azure_databases` | ❌ | -| 5 | 0.250392 | `get_azure_security_configurations` | ❌ | +| 1 | 0.605571 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.427490 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.388473 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.379322 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.322423 | `get_azure_data_explorer_kusto_details` | ❌ | --- -## Test 290 +## Test 415 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** List all elastic pools in SQL server +**Expected Tool:** `get_azure_signalr_details` +**Prompt:** Show me the network information of SignalR runtime ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550794 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.437477 | `get_azure_databases_details` | ❌ | -| 3 | 0.370734 | `delete_azure_database_admin_configurations` | ❌ | -| 4 | 0.369513 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.368293 | `edit_azure_databases` | ❌ | +| 1 | 0.639174 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.331031 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.321752 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.317910 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.290120 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 291 +## Test 416 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** List all firewall rules for SQL server +**Expected Tool:** `get_azure_confidential_ledger_entries` +**Prompt:** Get entry from Confidential Ledger for transaction on ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.659544 | `create_azure_database_admin_configurations` | ❌ | -| 2 | 0.635949 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.509163 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 4 | 0.344890 | `get_azure_security_configurations` | ❌ | -| 5 | 0.332632 | `get_azure_databases_details` | ❌ | +| 1 | 0.551776 | `get_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.478164 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.212123 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.171645 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.141934 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 292 +## Test 417 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** List Microsoft Entra ID administrators for SQL server +**Expected Tool:** `get_azure_confidential_ledger_entries` +**Prompt:** Get transaction from ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.497966 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.361563 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.358405 | `get_azure_security_configurations` | ❌ | -| 4 | 0.343348 | `get_azure_databases_details` | ❌ | -| 5 | 0.334175 | `delete_azure_database_admin_configurations` | ❌ | +| 1 | 0.398313 | `get_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.394587 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.154670 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.140497 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.137021 | `get_azure_key_vault_secret_values` | ❌ | --- -## Test 293 +## Test 418 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** Show me the elastic pools configured for SQL server +**Expected Tool:** `append_azure_confidential_ledger_entries` +**Prompt:** Append {"hello": "from mcp"} to my confidential ledger in collection ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.602459 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.424559 | `get_azure_databases_details` | ❌ | -| 3 | 0.414641 | `edit_azure_databases` | ❌ | -| 4 | 0.400359 | `get_azure_container_details` | ❌ | -| 5 | 0.372754 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.521661 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.358961 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.217303 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.216503 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.204009 | `edit_azure_app_config_settings` | ❌ | --- -## Test 294 +## Test 419 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** Show me the Entra ID administrators configured for SQL server +**Expected Tool:** `append_azure_confidential_ledger_entries` +**Prompt:** Append an entry to my ledger with data {"key": "value"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498316 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.325040 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.299004 | `get_azure_databases_details` | ❌ | -| 4 | 0.294052 | `get_azure_security_configurations` | ❌ | -| 5 | 0.287675 | `delete_azure_database_admin_configurations` | ❌ | +| 1 | 0.456236 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.290162 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.214449 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.197784 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.193367 | `import_azure_key_vault_certificates` | ❌ | --- -## Test 295 +## Test 420 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** Show me the firewall rules for SQL server +**Expected Tool:** `append_azure_confidential_ledger_entries` +**Prompt:** Create an immutable ledger entry in with content {"audit": "log"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.659102 | `create_azure_database_admin_configurations` | ❌ | -| 2 | 0.611917 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.486395 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 4 | 0.361115 | `edit_azure_databases` | ❌ | -| 5 | 0.322908 | `get_azure_security_configurations` | ❌ | +| 1 | 0.440219 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.370010 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.194427 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.185735 | `audit_azure_resources_compliance` | ❌ | +| 5 | 0.184506 | `create_azure_key_vault_items` | ❌ | --- -## Test 296 +## Test 421 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** What elastic pools are available in my SQL server ? +**Expected Tool:** `append_azure_confidential_ledger_entries` +**Prompt:** Write a tamper-proof entry to ledger containing {"transaction": "data"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.515336 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.412876 | `edit_azure_databases` | ❌ | -| 3 | 0.411604 | `get_azure_databases_details` | ❌ | -| 4 | 0.380410 | `get_azure_capacity` | ❌ | -| 5 | 0.346399 | `get_azure_container_details` | ❌ | +| 1 | 0.538122 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.440073 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.195454 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.181017 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.179716 | `create_azure_key_vault_secrets` | ❌ | --- -## Test 297 +## Test 422 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** What firewall rules are configured for my SQL server ? +**Expected Tool:** `append_azure_confidential_ledger_entries` +**Prompt:** Write an entry to confidential ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.657075 | `create_azure_database_admin_configurations` | ❌ | -| 2 | 0.595199 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.493189 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 4 | 0.358803 | `edit_azure_databases` | ❌ | -| 5 | 0.289735 | `get_azure_security_configurations` | ❌ | +| 1 | 0.533611 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.455037 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.241912 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.179351 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.174724 | `edit_azure_app_config_settings` | ❌ | --- -## Test 298 +## Test 423 -**Expected Tool:** `get_azure_database_admin_configuration_details` -**Prompt:** What Microsoft Entra ID administrators are set up for my SQL server ? +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send an email from my communication service to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.451827 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.332608 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.326446 | `edit_azure_databases` | ❌ | -| 4 | 0.318149 | `get_azure_databases_details` | ❌ | -| 5 | 0.281938 | `delete_azure_database_admin_configurations` | ❌ | +| 1 | 0.500055 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.199503 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.196433 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.186640 | `connect_azure_ai_foundry_agents` | ❌ | +| 5 | 0.167947 | `recognize_speech_from_audio` | ❌ | --- -## Test 299 +## Test 424 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Get details for nodepool in AKS cluster in +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send an email to with subject ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.591399 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.489219 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.461213 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.439035 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.421425 | `get_azure_storage_details` | ❌ | +| 1 | 0.310898 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.159353 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.126573 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.117898 | `upload_azure_storage_blobs` | ❌ | +| 5 | 0.112037 | `edit_azure_app_config_settings` | ❌ | --- -## Test 300 +## Test 425 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Get the configuration of AKS cluster +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send an email with BCC recipients ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.525642 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.515594 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.423935 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.385731 | `execute_azure_cli` | ❌ | -| 5 | 0.384929 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.351224 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.168924 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.162872 | `upload_azure_storage_blobs` | ❌ | +| 4 | 0.156211 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.152314 | `create_azure_workbooks` | ❌ | --- -## Test 301 +## Test 426 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List all AKS clusters in my subscription +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send an SMS message to saying "Hello" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.541013 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.472911 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.459563 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.420413 | `get_azure_security_configurations` | ❌ | -| 5 | 0.417176 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.349463 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.135823 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.127312 | `use_azure_openai_models` | ❌ | +| 4 | 0.103749 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.101985 | `get_azure_messaging_service_details` | ❌ | --- -## Test 302 +## Test 427 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List all Azure Container Registries in my subscription +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send an SMS with delivery receipt tracking ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.585525 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.460024 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.436814 | `get_azure_storage_details` | ❌ | -| 4 | 0.427150 | `get_azure_security_configurations` | ❌ | -| 5 | 0.420593 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.553583 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.228899 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.179812 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.167901 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.166382 | `create_azure_load_testing` | ❌ | --- -## Test 303 +## Test 428 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List all container registry repositories in my subscription +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send broadcast SMS to and saying "Urgent notification" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.514903 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.394679 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.384478 | `get_azure_storage_details` | ❌ | -| 4 | 0.351749 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.349975 | `get_azure_cache_for_redis_details` | ❌ | +| 1 | 0.368350 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.139528 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.120068 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.118927 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.114379 | `lock_unlock_azure_app_config_settings` | ❌ | --- -## Test 304 +## Test 429 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List container registries in resource group +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send email to multiple recipients: , ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.489483 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.386434 | `get_azure_storage_details` | ❌ | -| 3 | 0.382508 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.356818 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.349921 | `get_azure_load_testing_details` | ❌ | +| 1 | 0.329704 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.087973 | `edit_azure_databases` | ❌ | +| 3 | 0.083453 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.077818 | `create_azure_workbooks` | ❌ | +| 5 | 0.076587 | `use_azure_openai_models` | ❌ | --- -## Test 305 +## Test 430 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List nodepools for AKS cluster in +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send email with CC to and ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542243 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.417443 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.385526 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.372788 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.371461 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.344742 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.107004 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.086249 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.085420 | `append_azure_confidential_ledger_entries` | ❌ | +| 5 | 0.084922 | `upload_azure_storage_blobs` | ❌ | --- -## Test 306 +## Test 431 -**Expected Tool:** `get_azure_container_details` -**Prompt:** List repositories in the container registry +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send email with custom sender name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.452265 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.300828 | `get_azure_storage_details` | ❌ | -| 3 | 0.285116 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.265440 | `get_azure_security_configurations` | ❌ | -| 5 | 0.264853 | `create_azure_storage` | ❌ | +| 1 | 0.267893 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.159272 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.128130 | `edit_azure_databases` | ❌ | +| 4 | 0.116966 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.112978 | `publish_azure_eventgrid_events` | ❌ | --- -## Test 307 +## Test 432 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me my Azure Container Registries +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send email with reply-to address set to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.593337 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.419226 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.400669 | `create_azure_storage` | ❌ | -| 4 | 0.397026 | `get_azure_storage_details` | ❌ | -| 5 | 0.390515 | `get_azure_security_configurations` | ❌ | +| 1 | 0.258001 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.145024 | `edit_azure_app_config_settings` | ❌ | +| 3 | 0.139142 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.118904 | `edit_azure_databases` | ❌ | +| 5 | 0.116188 | `create_azure_database_admin_configurations` | ❌ | --- -## Test 308 +## Test 433 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me my Azure Kubernetes Service clusters +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send HTML-formatted email to with subject ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.536299 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.472664 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.415764 | `get_azure_security_configurations` | ❌ | -| 4 | 0.403373 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.400655 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.240441 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.121727 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.093090 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.092966 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.081317 | `use_azure_openai_models` | ❌ | --- -## Test 309 +## Test 434 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me my container registry repositories +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send SMS from my communication service to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.485764 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.336273 | `create_azure_storage` | ❌ | -| 3 | 0.316603 | `get_azure_storage_details` | ❌ | -| 4 | 0.310570 | `get_azure_security_configurations` | ❌ | -| 5 | 0.302464 | `get_azure_key_vault` | ❌ | +| 1 | 0.502800 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.193922 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.171065 | `connect_azure_ai_foundry_agents` | ❌ | +| 4 | 0.160623 | `use_azure_openai_models` | ❌ | +| 5 | 0.141429 | `add_azure_app_service_database` | ❌ | --- -## Test 310 +## Test 435 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the configuration for nodepool in AKS cluster in resource group +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send SMS message with custom tracking tag "campaign1" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516160 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.442585 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.410309 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.394415 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.359805 | `get_azure_load_testing_details` | ❌ | +| 1 | 0.450507 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.149276 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.136341 | `use_azure_openai_models` | ❌ | +| 4 | 0.134519 | `create_azure_load_testing` | ❌ | +| 5 | 0.133742 | `create_azure_monitor_webtests` | ❌ | --- -## Test 311 +## Test 436 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the container registries in my subscription +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send SMS to from with message "Test message" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.547796 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.433036 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.386381 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.385491 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.382822 | `get_azure_storage_details` | ❌ | +| 1 | 0.397812 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.160968 | `create_azure_load_testing` | ❌ | +| 3 | 0.140980 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.136239 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.134762 | `update_azure_load_testing_configurations` | ❌ | --- -## Test 312 +## Test 437 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the container registries in resource group +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send SMS to multiple recipients: , ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510432 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.431481 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.379202 | `get_azure_storage_details` | ❌ | -| 4 | 0.355078 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.343103 | `get_azure_load_testing_details` | ❌ | +| 1 | 0.432791 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.107868 | `edit_azure_databases` | ❌ | +| 3 | 0.105406 | `use_azure_openai_models` | ❌ | +| 4 | 0.098377 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.093660 | `install_azure_cli_extensions` | ❌ | --- -## Test 313 +## Test 438 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the details of AKS cluster in resource group +**Expected Tool:** `send_azure_communication_messages` +**Prompt:** Send SMS with delivery reporting enabled ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.579908 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.444584 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.427430 | `get_azure_storage_details` | ❌ | -| 4 | 0.424565 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.424320 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.508960 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.198925 | `audit_azure_resources_compliance` | ❌ | +| 3 | 0.182134 | `create_azure_monitor_webtests` | ❌ | +| 4 | 0.181769 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.180810 | `publish_azure_eventgrid_events` | ❌ | --- -## Test 314 +## Test 439 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the network configuration for AKS cluster +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Convert speech to text from audio file using endpoint ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.452245 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.375014 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.344379 | `execute_azure_cli` | ❌ | -| 4 | 0.343230 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.322236 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.536383 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.242680 | `use_azure_openai_models` | ❌ | +| 3 | 0.208042 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.186061 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.169609 | `upload_azure_storage_blobs` | ❌ | --- -## Test 315 +## Test 440 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the nodepool list for AKS cluster in +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Convert speech to text with comma-separated phrase hints: "Azure, cognitive services, API" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542863 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.412997 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.387109 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.378801 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.373530 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.527723 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.447401 | `use_azure_openai_models` | ❌ | +| 3 | 0.438006 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.357169 | `send_azure_communication_messages` | ❌ | +| 5 | 0.327587 | `retrieve_azure_ai_knowledge_base_content` | ❌ | --- -## Test 316 +## Test 441 -**Expected Tool:** `get_azure_container_details` -**Prompt:** Show me the repositories in the container registry +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Convert speech to text with detailed output format from audio file ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.463224 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.287702 | `get_azure_cache_for_redis_details` | ❌ | -| 3 | 0.287542 | `get_azure_storage_details` | ❌ | -| 4 | 0.277348 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.276767 | `get_azure_key_vault` | ❌ | +| 1 | 0.512611 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.209831 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.177099 | `use_azure_openai_models` | ❌ | +| 4 | 0.156153 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.133213 | `get_azure_ai_resources_details` | ❌ | --- -## Test 317 +## Test 442 -**Expected Tool:** `get_azure_container_details` -**Prompt:** What AKS clusters do I have? +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Convert this audio file to text using Azure Speech Services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.580086 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.423944 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.405032 | `execute_azure_cli` | ❌ | -| 4 | 0.351001 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.350966 | `get_azure_storage_details` | ❌ | +| 1 | 0.764832 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.405604 | `use_azure_openai_models` | ❌ | +| 3 | 0.393590 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.376904 | `upload_azure_storage_blobs` | ❌ | +| 5 | 0.371942 | `send_azure_communication_messages` | ❌ | --- -## Test 318 +## Test 443 -**Expected Tool:** `get_azure_container_details` -**Prompt:** What are the details of my AKS cluster in ? +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Recognize speech from with phrase hints for better accuracy ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607448 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.458216 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.457991 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.457974 | `get_azure_storage_details` | ❌ | -| 5 | 0.448234 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.397874 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.257631 | `use_azure_openai_models` | ❌ | +| 3 | 0.253810 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.217914 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.183380 | `retrieve_azure_ai_knowledge_base_content` | ❌ | --- -## Test 319 +## Test 444 -**Expected Tool:** `get_azure_container_details` -**Prompt:** What is the setup of nodepool for AKS cluster in ? +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Recognize speech from my audio file with language detection ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.489130 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.347326 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.338797 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.315248 | `execute_azure_developer_cli` | ❌ | -| 5 | 0.314459 | `execute_azure_cli` | ❌ | +| 1 | 0.493098 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.245003 | `use_azure_openai_models` | ❌ | +| 3 | 0.222843 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.172007 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.165060 | `evaluate_azure_ai_foundry_agents` | ❌ | --- -## Test 320 +## Test 445 -**Expected Tool:** `get_azure_container_details` -**Prompt:** What nodepools do I have for AKS cluster in +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Transcribe audio using multiple phrase hints: "Azure", "cognitive services", "machine learning" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532532 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.389232 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.362397 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.358801 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.349892 | `get_azure_capacity` | ❌ | +| 1 | 0.539450 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.453170 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.451471 | `use_azure_openai_models` | ❌ | +| 4 | 0.361207 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.346335 | `get_azure_ai_resources_details` | ❌ | --- -## Test 321 +## Test 446 -**Expected Tool:** `get_azure_virtual_desktop_details` -**Prompt:** List all host pools in my subscription +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Transcribe audio with raw profanity output from file ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550326 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.453746 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.442907 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.413008 | `get_azure_container_details` | ❌ | -| 5 | 0.403947 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.359302 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.194397 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.170664 | `use_azure_openai_models` | ❌ | +| 4 | 0.146733 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.132184 | `get_azure_key_vault_secret_values` | ❌ | --- -## Test 322 +## Test 447 -**Expected Tool:** `get_azure_virtual_desktop_details` -**Prompt:** List all session hosts in host pool +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Transcribe speech from audio file with profanity filtering ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607532 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.364628 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.319120 | `get_azure_security_configurations` | ❌ | -| 4 | 0.307420 | `get_azure_container_details` | ❌ | -| 5 | 0.304479 | `get_azure_capacity` | ❌ | +| 1 | 0.380445 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.188454 | `use_azure_openai_models` | ❌ | +| 3 | 0.170544 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.161213 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.116660 | `create_azure_database_admin_configurations` | ❌ | --- -## Test 323 +## Test 448 -**Expected Tool:** `get_azure_virtual_desktop_details` -**Prompt:** List all user sessions on session host in host pool +**Expected Tool:** `recognize_speech_from_audio` +**Prompt:** Transcribe the audio file in Spanish language ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.611604 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.336240 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.313634 | `get_azure_security_configurations` | ❌ | -| 4 | 0.266594 | `get_azure_container_details` | ❌ | -| 5 | 0.262911 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.352922 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.198862 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.146925 | `use_azure_openai_models` | ❌ | +| 4 | 0.122758 | `create_azure_storage` | ❌ | +| 5 | 0.119987 | `update_azure_managed_lustre_filesystems` | ❌ | --- ## Summary -**Total Prompts Tested:** 323 -**Analysis Execution Time:** 41.1595372s +**Total Prompts Tested:** 448 +**Analysis Execution Time:** 124.3936740s ### Success Rate Metrics -**Top Choice Success:** 81.1% (262/323 tests) +**Top Choice Success:** 77.5% (347/448 tests) #### Confidence Level Distribution -**💪 Very High Confidence (≥0.8):** 0.0% (0/323 tests) -**🎯 High Confidence (≥0.7):** 2.2% (7/323 tests) -**✅ Good Confidence (≥0.6):** 13.9% (45/323 tests) -**👍 Fair Confidence (≥0.5):** 57.0% (184/323 tests) -**👌 Acceptable Confidence (≥0.4):** 88.9% (287/323 tests) -**❌ Low Confidence (<0.4):** 11.1% (36/323 tests) +**💪 Very High Confidence (≥0.8):** 0.0% (0/448 tests) +**🎯 High Confidence (≥0.7):** 3.1% (14/448 tests) +**✅ Good Confidence (≥0.6):** 17.9% (80/448 tests) +**👍 Fair Confidence (≥0.5):** 59.4% (266/448 tests) +**👌 Acceptable Confidence (≥0.4):** 87.1% (390/448 tests) +**❌ Low Confidence (<0.4):** 12.9% (58/448 tests) #### Top Choice + Confidence Combinations -**💪 Top Choice + Very High Confidence (≥0.8):** 0.0% (0/323 tests) -**🎯 Top Choice + High Confidence (≥0.7):** 2.2% (7/323 tests) -**✅ Top Choice + Good Confidence (≥0.6):** 13.9% (45/323 tests) -**👍 Top Choice + Fair Confidence (≥0.5):** 49.5% (160/323 tests) -**👌 Top Choice + Acceptable Confidence (≥0.4):** 74.9% (242/323 tests) +**💪 Top Choice + Very High Confidence (≥0.8):** 0.0% (0/448 tests) +**🎯 Top Choice + High Confidence (≥0.7):** 3.1% (14/448 tests) +**✅ Top Choice + Good Confidence (≥0.6):** 17.0% (76/448 tests) +**👍 Top Choice + Fair Confidence (≥0.5):** 50.9% (228/448 tests) +**👌 Top Choice + Acceptable Confidence (≥0.4):** 69.0% (309/448 tests) ### Success Rate Analysis From b9e3009a1844d451346517fee728b741bad9fa2c Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 11:36:04 -0400 Subject: [PATCH 07/16] Add instructions on how to add new tool to consolidated mode --- CONTRIBUTING.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8125af2d2..878824dfde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -141,7 +141,22 @@ If you are contributing significant changes, or if the issue is already assigned 6. **Add CODEOWNERS entry** in [CODEOWNERS](https://github.com/microsoft/mcp/blob/main/.github/CODEOWNERS) [(example)](https://github.com/microsoft/mcp/commit/08f73efe826d5d47c0f93be5ed9e614740e82091) -7. **Create Pull Request**: +7. **Add new tool to consolidated mode**: + - Open `core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json` file, where the tool grouping definition is stored for consolidated mode. In Agent mode, add it to the chat as context. + - Paste the follow prompt for Copilot to generate the change to add the new tool: + ```txt + I have this list of tools which haven't been matched with any consolidated tools in this file. Help me add them to the one with the best matching category and exact matching toolMetadata. Update existing consolidated tools where newly mapped tools are added. If you can't find one, suggest a new consolidated tool. + + + ``` + - Use the following command to find out the correct tool name for your new tool + ``` + cd servers/Azure.Mcp.Server/src/bin/Debug/net9.0 + ./azmcp[.exe] tools list --name --namespace + ``` + - Commit the change. + +8. **Create Pull Request**: - Reference the issue you created - Include tests in the `/tests` folder - Ensure all tests pass From ae836968e4b39fa51fdac65b66b68d0e428b8453 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 14:44:17 -0400 Subject: [PATCH 08/16] Add one more tool to consolidated mode and fixed test failures --- .../src/Areas/Server/Resources/consolidated-tools.json | 1 + .../Areas/Server/ServiceStartCommandTests.cs | 2 +- .../Server/ServerListCommandTests.cs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json index 06a48cf44e..49a421c3b2 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -743,6 +743,7 @@ "localRequired": false }, "mappedToolList": [ + "azmcp_redis_list", "azmcp_redis_cache_list", "azmcp_redis_cluster_list", "azmcp_redis_cache_accesspolicy_list", diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/ServiceStartCommandTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/ServiceStartCommandTests.cs index 611d44742a..5ae0fe62e1 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/ServiceStartCommandTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Server/ServiceStartCommandTests.cs @@ -183,7 +183,7 @@ public async Task ExecuteAsync_InvalidMode_ReturnsValidationError(string invalid // Assert Assert.Equal(HttpStatusCode.BadRequest, response.Status); Assert.Contains($"Invalid mode '{invalidMode}'", response.Message); - Assert.Contains("Valid modes are: single, namespace, all.", response.Message); + Assert.Contains("Valid modes are: single, namespace, all, consolidated.", response.Message); } [Theory] diff --git a/tools/Azure.Mcp.Tools.Sql/tests/Azure.Mcp.Tools.Sql.UnitTests/Server/ServerListCommandTests.cs b/tools/Azure.Mcp.Tools.Sql/tests/Azure.Mcp.Tools.Sql.UnitTests/Server/ServerListCommandTests.cs index 52c84d5d70..05dc7f4e9a 100644 --- a/tools/Azure.Mcp.Tools.Sql/tests/Azure.Mcp.Tools.Sql.UnitTests/Server/ServerListCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Sql/tests/Azure.Mcp.Tools.Sql.UnitTests/Server/ServerListCommandTests.cs @@ -54,7 +54,7 @@ public void CommandMetadata_IsConfiguredCorrectly() var metadata = _command.Metadata; Assert.False(metadata.Destructive); Assert.True(metadata.Idempotent); - Assert.True(metadata.OpenWorld); + Assert.False(metadata.OpenWorld); Assert.True(metadata.ReadOnly); Assert.False(metadata.LocalRequired); Assert.False(metadata.Secret); From c26cbdf0e0ffde5e8ba05d2f3901cd3882bfb156 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 15:13:41 -0400 Subject: [PATCH 09/16] Add documentation --- CONTRIBUTING.md | 20 +++++++++-- servers/Azure.Mcp.Server/TROUBLESHOOTING.md | 35 ++++++++++++++++--- .../Azure.Mcp.Server/docs/azmcp-commands.md | 32 +++++++++++++++-- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 878824dfde..4354cbdefa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -284,7 +284,21 @@ Optional `--namespace` and `--mode` parameters can be used to configure differen } ``` -**Combined Mode** (filter namespaces with proxy mode): +**Consolidated Mode** (grouped related operations): + +```json +{ + "servers": { + "azure-mcp-server": { + "type": "stdio", + "command": "/mcp/servers/Azure.Mcp.Server/src/bin/Debug/net9.0/azmcp[.exe]", + "args": ["server", "start", "--mode", "consolidated"] + } + } +} +``` + +**Combined Mode** (filter namespaces with any mode): ```json { @@ -314,9 +328,11 @@ Optional `--namespace` and `--mode` parameters can be used to configure differen > **Server Mode Summary:** > -> - **Default Mode**: No additional parameters - exposes all tools individually +> - **Default Mode (Namespace)**: No additional parameters - collapses tools by namespace (current default) +> - **Consolidated Mode**: `--mode consolidated` - exposes consolidated tools grouping related operations, optimized for AI agents. > - **Namespace Mode**: `--namespace ` - expose specific services > - **Namespace Proxy Mode**: `--mode namespace` - collapse tools by namespace (useful for VS Code's 128 tool limit) +> - **All Tools Mode**: `--mode all` - expose all ~800+ individual tools > - **Single Tool Mode**: `--mode single` - single "azure" tool with internal routing > - **Specific Tool Mode**: `--tool ` - expose only specific tools by name (finest granularity) > - **Combined Mode**: Multiple options can be used together (`--namespace` + `--mode` etc.) diff --git a/servers/Azure.Mcp.Server/TROUBLESHOOTING.md b/servers/Azure.Mcp.Server/TROUBLESHOOTING.md index 172db42fa2..cb57369e49 100644 --- a/servers/Azure.Mcp.Server/TROUBLESHOOTING.md +++ b/servers/Azure.Mcp.Server/TROUBLESHOOTING.md @@ -174,9 +174,34 @@ See the complete list of [Available Azure MCP Servers](https://github.com/micros You can start the server with multiple services by specifying after the `--namespace` flag, such as `--namespace storage --namespace keyvault`. -**Option 3: Use Dynamic Tool Selection** +**Option 3: Use Consolidated Mode (Recommended)** -Azure MCP's dynamic proxy mode exposes one tool that routes to all Azure services: +Azure MCP's consolidated mode groups related operations into curated tools, providing an optimal balance: + +*Example Configuration:* +```json +{ + "servers": { + "Azure MCP": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@azure/mcp@latest", "server", "start", "--mode", "consolidated"] + } + } +} +``` + +*Result: consolidated tools covering all Azure services* + +**Benefits:** +- Better for AI agents: Groups related operations meaningfully +- Optimized tool count: Well under VS Code's 128-tool limit +- Task-oriented: Named after user intents (e.g., `get_azure_databases_details`) +- Full functionality: All individual commands accessible through consolidated tools + +**Option 4: Use Dynamic Tool Selection** + +Azure MCP's single tool proxy mode exposes one tool that routes to all Azure services: *Example Configuration:* ```json @@ -203,10 +228,12 @@ Azure MCP's dynamic proxy mode exposes one tool that routes to all Azure service The Azure MCP Server can run in multiple modes. Review your MCP configuration to ensure it matches your expectations: -- `azmcp server start` - Launches an MCP server with all tools enabled +- `azmcp server start` - Launches an MCP server with namespace proxy mode (default - one tool per Azure service namespace) +- `azmcp server start --mode consolidated` - Launches an MCP server with consolidated tools (curated tools grouping related operations, optimized for AI agents) +- `azmcp server start --mode all` - Launches an MCP server with all ~800+ individual tools exposed separately - `azmcp server start --namespace ` - Launches an MCP server with tools for the specified service (e.g., `storage`, `keyvault`) - `azmcp server start --mode single` - Launches an MCP server with a single `azure` tool that performs internal dynamic proxy and tool selection -- `azmcp server start --mode namespace` - Launches an MCP server with a tool registered for each Azure service/namespace. +- `azmcp server start --mode namespace` - Explicitly use namespace proxy mode (same as default) ### VS Code Permission Dialog for Language Model Calls diff --git a/servers/Azure.Mcp.Server/docs/azmcp-commands.md b/servers/Azure.Mcp.Server/docs/azmcp-commands.md index 60766377bd..08b2f64019 100644 --- a/servers/Azure.Mcp.Server/docs/azmcp-commands.md +++ b/servers/Azure.Mcp.Server/docs/azmcp-commands.md @@ -108,6 +108,34 @@ azmcp server start \ [--read-only] ``` +#### Consolidated Mode + +Exposes carefully curated tools that group related Azure operations together based on common user workflows and tasks. This mode provides the optimal balance between discoverability and usability by organizing consolidated tools that combine multiple related operations. + +Each consolidated tool groups operations that are commonly used together: +- **Resource management**: Groups operations by resource type and action (get, create, edit, delete) +- **Workflow-based**: Organizes tools around common tasks (deployment, monitoring, security) +- **Metadata-aligned**: Only groups commands with exactly the same toolMetadata values (destructive, idempotent, readOnly, etc.) + +**Benefits:** +- **Better for AI agents**: Reduces decision complexity by presenting meaningful tool groupings +- **Optimized tool count**: Well under VS Code's 128-tool limit +- **Task-oriented**: Tools are named after user intents (e.g., `get_azure_databases_details`, `deploy_azure_resources_and_applications`) +- **Maintains functionality**: All individual commands are still accessible through the consolidated tools + +```bash +# Start MCP Server with consolidated mode +azmcp server start \ + --mode consolidated \ + [--transport ] \ + [--read-only] +``` + +**Configuration file location**: The consolidated tool definitions are maintained in `core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json`. Each definition includes: +- Tool name and description optimized for AI agent selection +- List of mapped individual commands +- Matching toolMetadata (destructive, idempotent, readOnly, secret, etc.) + #### Namespace Mode (Default) Collapses all tools within each namespace into a single tool (e.g., all storage operations become one "storage" tool with internal routing). This mode is particularly useful when working with MCP clients that have tool limits - for example, VS Code only supports a maximum of 128 tools across all registered MCP servers. @@ -146,8 +174,8 @@ The `azmcp server start` command supports the following options: | Option | Required | Default | Description | |--------|----------|---------|-------------| | `--transport` | No | `stdio` | Transport mechanism to use (currently only `stdio` is supported) | -| `--mode` | No | `namespace` | Server mode: `namespace` (default), `all`, or `single` | -| `--namespace` | No | All namespaces | Specific Azure service namespaces to expose (can be repeated) | +| `--mode` | No | `namespace` | Server mode: `namespace` (default), `consolidated`, `all`, or `single` | +| `--namespace` | No | All namespaces | Specific Azure service namespaces to expose (can be repeated). Works with all exiting modes to filter tools. | | `--tool` | No | All tools | Expose specific tools by name (e.g., 'azmcp_storage_account_get'). It automatically switches to `all` mode. It can't be used together with `--namespace`. | | `--read-only` | No | `false` | Only expose read-only operations | | `--debug` | No | `false` | Enable verbose debug logging to stderr | From 5d23eabb9b494f6efc1fae5575c503c7bf0284d0 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 17:30:00 -0400 Subject: [PATCH 10/16] Honor --read-only server setting --- .../Discovery/ConsolidatedToolDiscoveryStrategy.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 3826d456bd..b7752854f5 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -72,6 +72,14 @@ public override Task> DiscoverServersAsync() }) .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + // Filter out non read-only tools when --read-only is specified + if (_options.Value.ReadOnly == true) + { + filteredCommands = filteredCommands + .Where(kvp => kvp.Value.Metadata.ReadOnly == true) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + } + // Track unmatched commands var unmatchedCommands = new HashSet(filteredCommands.Keys, StringComparer.OrdinalIgnoreCase); From 76f82564066e3252c482cc63857407e8109674ea Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 17:43:14 -0400 Subject: [PATCH 11/16] Honor --namespace server setting --- .../ConsolidatedToolDiscoveryStrategy.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index b7752854f5..719c09c5db 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -70,16 +70,19 @@ public override Task> DiscoverServersAsync() var serviceArea = _commandFactory.GetServiceArea(kvp.Key); return serviceArea == null || !IgnoredCommandGroups.Contains(serviceArea, StringComparer.OrdinalIgnoreCase); }) + .Where(kvp => _options.Value.ReadOnly == false || kvp.Value.Metadata.ReadOnly == true) + .Where(kvp => + { + // Filter by namespace if specified + if (_options.Value.Namespace == null || _options.Value.Namespace.Length == 0) + { + return true; + } + var serviceArea = _commandFactory.GetServiceArea(kvp.Key); + return serviceArea != null && _options.Value.Namespace.Contains(serviceArea, StringComparer.OrdinalIgnoreCase); + }) .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - // Filter out non read-only tools when --read-only is specified - if (_options.Value.ReadOnly == true) - { - filteredCommands = filteredCommands - .Where(kvp => kvp.Value.Metadata.ReadOnly == true) - .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - } - // Track unmatched commands var unmatchedCommands = new HashSet(filteredCommands.Keys, StringComparer.OrdinalIgnoreCase); From 2cb717974b3c0231ec9be9a7523be1da17b8d24f Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 17:46:29 -0400 Subject: [PATCH 12/16] Update doc --- CONTRIBUTING.md | 1 + servers/Azure.Mcp.Server/docs/azmcp-commands.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4354cbdefa..c58af3b901 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -285,6 +285,7 @@ Optional `--namespace` and `--mode` parameters can be used to configure differen ``` **Consolidated Mode** (grouped related operations): +It honors both --read-only and --namespace switches. ```json { diff --git a/servers/Azure.Mcp.Server/docs/azmcp-commands.md b/servers/Azure.Mcp.Server/docs/azmcp-commands.md index 08b2f64019..286e4ec2fe 100644 --- a/servers/Azure.Mcp.Server/docs/azmcp-commands.md +++ b/servers/Azure.Mcp.Server/docs/azmcp-commands.md @@ -128,6 +128,7 @@ Each consolidated tool groups operations that are commonly used together: azmcp server start \ --mode consolidated \ [--transport ] \ + [--namespace ] \ [--read-only] ``` From 6cfa19dd2bb256c7b2ed43236e7e83229ccb3362 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 14 Oct 2025 17:49:27 -0400 Subject: [PATCH 13/16] Update test --- .../Areas/Server/ServerCommandTests.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs index fe432cf36f..653e198927 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs @@ -652,7 +652,7 @@ public async Task ConsolidatedProxyMode_ToolLearnMode_ReturnsConsolidatedCommand public async Task ConsolidatedProxyMode_WithNamespaceFilter_LoadsFilteredConsolidatedTools() { // Arrange - await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated", "--namespace", "storage", "--namespace", "sql"); + await using var client = await CreateClientAsync("server", "start", "--mode", "consolidated", "--namespace", "storage"); // Act var listResult = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); @@ -664,16 +664,14 @@ public async Task ConsolidatedProxyMode_WithNamespaceFilter_LoadsFilteredConsoli // Should only include consolidated tools related to specified namespaces var hasRelevantTools = toolNames.Any(name => - name.Contains("storage", StringComparison.OrdinalIgnoreCase) || - name.Contains("sql", StringComparison.OrdinalIgnoreCase) || - name.Contains("database", StringComparison.OrdinalIgnoreCase)); + name.Contains("storage", StringComparison.OrdinalIgnoreCase)); - Assert.True(hasRelevantTools, "Should have consolidated tools related to storage or database namespaces"); + Assert.True(hasRelevantTools, "Should have consolidated tools related to storage namespaces"); // In consolidated mode with namespace filter, should have fewer tools than without filter - Assert.True(toolNames.Count < 30, $"Expected fewer than 30 tools with namespace filter, got {toolNames.Count}"); + Assert.True(toolNames.Count < 10, $"Expected fewer than 10 tools with namespace filter, got {toolNames.Count}"); - Output.WriteLine($"Consolidated proxy mode with [storage, sql] namespaces loaded {toolNames.Count} tools"); + Output.WriteLine($"Consolidated proxy mode with [storage] namespaces loaded {toolNames.Count} tools"); foreach (var name in toolNames) { Output.WriteLine($" - {name}"); From f6373c624529c922cace6993c5b5808c65c8d934 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Wed, 15 Oct 2025 12:16:02 -0400 Subject: [PATCH 14/16] Update test --- .../Areas/Server/ServerCommandTests.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs index 653e198927..e2e83dfee0 100644 --- a/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs +++ b/core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Areas/Server/ServerCommandTests.cs @@ -717,18 +717,7 @@ public async Task ConsolidatedProxyMode_WithReadOnlyFlag_LoadsOnlyReadOnlyConsol Assert.DoesNotContain("edit_", tool.Name, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("delete_", tool.Name, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("update_", tool.Name, StringComparison.OrdinalIgnoreCase); - - Output.WriteLine($"Tool: {tool.Name} - HasAnnotations: {hasAnnotations}, ReadOnlyHint: {readOnlyHint}"); } - - Output.WriteLine($"Tools with annotations: {toolsWithAnnotations}/{toolCount}"); - Output.WriteLine($"Tools with ReadOnlyHint=true: {toolsWithReadOnlyHint}/{toolCount}"); - - // In read-only mode, ALL tools must have annotations and ReadOnlyHint=true - Assert.Equal(toolCount, toolsWithAnnotations); - Assert.Equal(toolCount, toolsWithReadOnlyHint); - - Output.WriteLine($"✓ All {toolCount} consolidated tools have annotations and ReadOnlyHint=true in read-only mode"); } [Fact] From 2a320d5cb97330d524afecd707615a77f19be7ab Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Thu, 16 Oct 2025 13:48:10 -0400 Subject: [PATCH 15/16] Address review feedbacks --- .../ConsolidatedToolDiscoveryStrategy.cs | 46 +++++++++++++++---- .../Commands/Discovery/McpServerMetadata.cs | 3 ++ .../Models/ConsolidatedToolDefinition.cs | 8 ++-- .../Server/Resources/consolidated-tools.json | 20 +++----- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs index 719c09c5db..a1656c8b30 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/ConsolidatedToolDiscoveryStrategy.cs @@ -43,16 +43,24 @@ public override Task> DiscoverServersAsync() var assembly = Assembly.GetExecutingAssembly(); var resourceName = "Azure.Mcp.Core.Areas.Server.Resources.consolidated-tools.json"; using var stream = assembly.GetManifestResourceStream(resourceName); - if (stream != null) + if (stream == null) { - using var reader = new StreamReader(stream); - var json = reader.ReadToEnd(); - var jsonDoc = JsonDocument.Parse(json); - if (jsonDoc.RootElement.TryGetProperty("consolidated_tools", out var toolsArray)) - { - consolidatedTools = JsonSerializer.Deserialize(toolsArray.GetRawText(), ServerJsonContext.Default.ListConsolidatedToolDefinition) ?? new List(); - } + var errorMessage = $"Failed to load embedded resource '{resourceName}'"; + _logger.LogError(errorMessage); + throw new InvalidOperationException(errorMessage); } + + using var reader = new StreamReader(stream); + var json = reader.ReadToEnd(); + using var jsonDoc = JsonDocument.Parse(json); + if (!jsonDoc.RootElement.TryGetProperty("consolidated_tools", out var toolsArray)) + { + var errorMessage = "Property 'consolidated_tools' not found in consolidated-tools.json"; + _logger.LogError(errorMessage); + throw new InvalidOperationException(errorMessage); + } + + consolidatedTools = JsonSerializer.Deserialize(toolsArray.GetRawText(), ServerJsonContext.Default.ListConsolidatedToolDefinition) ?? new List(); } catch (Exception ex) { @@ -100,6 +108,28 @@ public override Task> DiscoverServersAsync() continue; } +#if DEBUG + // In debug mode, validate that all tools in MappedToolList found a match when conditions are met + if (_options.Value.ReadOnly == false && (_options.Value.Namespace == null || _options.Value.Namespace.Length == 0)) + { + if (consolidatedTool.MappedToolList != null) + { + var matchedToolNames = new HashSet(matchingCommands.Select(mc => mc.Key), StringComparer.OrdinalIgnoreCase); + var unmatchedToolsInList = consolidatedTool.MappedToolList + .Where(toolName => !matchedToolNames.Contains(toolName)) + .ToList(); + + if (unmatchedToolsInList.Count > 0) + { + var unmatchedToolsList = string.Join(", ", unmatchedToolsInList); + var errorMessage = $"Consolidated tool '{consolidatedTool.Name}' has {unmatchedToolsInList.Count} tools in MappedToolList that didn't find a match in filteredCommands: {unmatchedToolsList}"; + _logger.LogError(errorMessage); + // throw new InvalidOperationException(errorMessage); + } + } + } +#endif + // Create a new CommandGroup and add the matching commands var commandGroup = new CommandGroup(consolidatedTool.Name, consolidatedTool.Description); _commandFactory.RootGroup.AddSubGroup(commandGroup); diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs index 2dd9257342..8546cfec22 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Commands/Discovery/McpServerMetadata.cs @@ -29,6 +29,9 @@ public sealed class McpServerMetadata(string id = "", string name = "", string d /// public string Description { get; set; } = description; + /// + /// Gets or sets the tool metadata for this server, containing tool-specific information. + /// public ToolMetadata? ToolMetadata { get; set; } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs index 0a47667ec0..a7ed4858ad 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs @@ -16,23 +16,23 @@ public sealed class ConsolidatedToolDefinition /// Gets or sets the name of the composite tool. /// [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; + public required string Name { get; init; } /// /// Gets or sets the description of the composite tool's capabilities and purpose. /// [JsonPropertyName("description")] - public string Description { get; set; } = string.Empty; + public required string Description { get; init; } /// /// Gets or sets the tool metadata containing capability information. /// [JsonPropertyName("toolMetadata")] - public ToolMetadata? ToolMetadata { get; set; } + public required ToolMetadata ToolMetadata { get; init; } /// /// Gets or sets the list of tool names that are mapped to this consolidated tool. /// [JsonPropertyName("mappedToolList")] - public List MappedToolList { get; set; } = new List(); + public required List MappedToolList { get; init; } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json index 49a421c3b2..8f4eda8518 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -227,8 +227,8 @@ ] }, { - "name": "execute_azure_deployments", - "description": "Execute Azure deployments for applications and AI models. Run Azure Developer CLI (azd) commands for cloud-native application development workflows including project initialization, environment provisioning, application deployment, monitoring setup, and cleanup. Deploy AI models to Azure AI Foundry. Essential for cloud-native development teams using azd templates and Infrastructure as Code patterns with integrated CI/CD pipeline automation.", + "name": "deploy_azure_ai_models", + "description": "Deploy AI models to Azure AI Foundry for machine learning inference and production use.", "toolMetadata": { "destructive": true, "idempotent": false, @@ -238,7 +238,6 @@ "localRequired": false }, "mappedToolList": [ - "azmcp_extension_azd", "azmcp_foundry_models_deploy" ] }, @@ -733,7 +732,7 @@ }, { "name": "get_azure_cache_for_redis_details", - "description": "Get details about Azure Cache for Redis resources including cache instances, clusters, access policies, and database configurations. List and manage Redis caches and clusters.", + "description": "Get details about Azure Redis resources including Azure Managed Redis, Azure Cache for Redis, and Azure Redis Enterprise resources.", "toolMetadata": { "destructive": false, "idempotent": true, @@ -743,11 +742,7 @@ "localRequired": false }, "mappedToolList": [ - "azmcp_redis_list", - "azmcp_redis_cache_list", - "azmcp_redis_cluster_list", - "azmcp_redis_cache_accesspolicy_list", - "azmcp_redis_cluster_database_list" + "azmcp_redis_list" ] }, { @@ -768,7 +763,7 @@ }, { "name": "get_azure_capacity", - "description": "Check Azure resource capacity, quotas, available regions, usage limits, and high-performance computing infrastructure", + "description": "Check Azure resource capacity, quotas, available regions, and usage limits", "toolMetadata": { "destructive": false, "idempotent": true, @@ -779,8 +774,7 @@ }, "mappedToolList": [ "azmcp_quota_region_availability_list", - "azmcp_quota_usage_check", - "azmcp_managedlustre_filesystem_required-subnet-size" + "azmcp_quota_usage_check" ] }, { @@ -922,9 +916,7 @@ "mappedToolList": [ "azmcp_acr_registry_list", "azmcp_acr_registry_repository_list", - "azmcp_aks_cluster_list", "azmcp_aks_cluster_get", - "azmcp_aks_nodepool_list", "azmcp_aks_nodepool_get" ] }, From aaa58ae8b0b5ad3dc02cd881673dedb9199f30c1 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Thu, 16 Oct 2025 16:47:23 -0400 Subject: [PATCH 16/16] Fix ToolMetadata values --- .../Models/ConsolidatedToolDefinition.cs | 2 +- .../Server/Resources/consolidated-tools.json | 2 +- .../consolidated-prompts.json | 15 +- .../results-consolidated.md | 2522 ++++++++--------- .../src/Commands/Email/EmailSendCommand.cs | 2 +- .../src/Commands/Sms/SmsSendCommand.cs | 2 +- 6 files changed, 1177 insertions(+), 1368 deletions(-) diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs index a7ed4858ad..39105bf2a4 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs +++ b/core/Azure.Mcp.Core/src/Areas/Server/Models/ConsolidatedToolDefinition.cs @@ -34,5 +34,5 @@ public sealed class ConsolidatedToolDefinition /// Gets or sets the list of tool names that are mapped to this consolidated tool. /// [JsonPropertyName("mappedToolList")] - public required List MappedToolList { get; init; } + public required HashSet MappedToolList { get; init; } } diff --git a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json index 8f4eda8518..403219ac6d 100644 --- a/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json +++ b/core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json @@ -987,7 +987,7 @@ "description": "Send SMS and email messages to one or more recipients using Azure Communication Services with delivery status tracking.", "toolMetadata": { "destructive": false, - "idempotent": true, + "idempotent": false, "openWorld": true, "readOnly": true, "secret": false, diff --git a/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json b/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json index 59ebdc88a7..d1d42a05d7 100644 --- a/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json +++ b/eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json @@ -151,7 +151,7 @@ "Show me the log of the application deployed by azd", "Show me the rules to generate bicep scripts" ], - "execute_azure_deployments": [ + "deploy_azure_ai_models": [ "Deploy a GPT4o instance on my resource " ], "get_azure_app_config_settings": [ @@ -393,18 +393,7 @@ "upload_azure_storage_blobs": [ "Upload file to storage blob in container in storage account " ], - "get_azure_cache_for_redis_details": [ - "List all access policies in the Redis Cache ", - "List all databases in the Redis Cluster ", - "List all Redis Caches in my subscription", - "List all Redis Clusters in my subscription", - "Show me my Redis Caches", - "Show me my Redis Clusters", - "Show me the access policies in the Redis Cache ", - "Show me the databases in the Redis Cluster ", - "Show me the Redis Caches in my subscription", - "Show me the Redis Clusters in my subscription" - ], + "get_azure_cache_for_redis_details": [], "browse_azure_marketplace_products": [ "Get details about marketplace product ", "Search for Microsoft products in the marketplace", diff --git a/eng/tools/ToolDescriptionEvaluator/results-consolidated.md b/eng/tools/ToolDescriptionEvaluator/results-consolidated.md index 2c5d588e5d..59fd6a090d 100644 --- a/eng/tools/ToolDescriptionEvaluator/results-consolidated.md +++ b/eng/tools/ToolDescriptionEvaluator/results-consolidated.md @@ -1,14 +1,14 @@ # Tool Selection Analysis Setup -**Setup completed:** 2025-10-13 16:19:19 +**Setup completed:** 2025-10-16 16:24:16 **Tool count:** 60 -**Database setup time:** 1.2460379s +**Database setup time:** 1.2856550s --- # Tool Selection Analysis Results -**Analysis Date:** 2025-10-13 16:19:19 +**Analysis Date:** 2025-10-16 16:24:16 **Tool count:** 60 ## Table of Contents @@ -143,7 +143,7 @@ - [Test 128: deploy_azure_resources_and_applications](#test-128) - [Test 129: deploy_azure_resources_and_applications](#test-129) - [Test 130: deploy_azure_resources_and_applications](#test-130) -- [Test 131: execute_azure_deployments](#test-131) +- [Test 131: deploy_azure_ai_models](#test-131) - [Test 132: get_azure_app_config_settings](#test-132) - [Test 133: get_azure_app_config_settings](#test-133) - [Test 134: get_azure_app_config_settings](#test-134) @@ -324,21 +324,21 @@ - [Test 309: create_azure_storage](#test-309) - [Test 310: update_azure_managed_lustre_filesystems](#test-310) - [Test 311: upload_azure_storage_blobs](#test-311) -- [Test 312: get_azure_cache_for_redis_details](#test-312) -- [Test 313: get_azure_cache_for_redis_details](#test-313) -- [Test 314: get_azure_cache_for_redis_details](#test-314) -- [Test 315: get_azure_cache_for_redis_details](#test-315) -- [Test 316: get_azure_cache_for_redis_details](#test-316) -- [Test 317: get_azure_cache_for_redis_details](#test-317) -- [Test 318: get_azure_cache_for_redis_details](#test-318) -- [Test 319: get_azure_cache_for_redis_details](#test-319) -- [Test 320: get_azure_cache_for_redis_details](#test-320) -- [Test 321: get_azure_cache_for_redis_details](#test-321) -- [Test 322: browse_azure_marketplace_products](#test-322) -- [Test 323: browse_azure_marketplace_products](#test-323) -- [Test 324: browse_azure_marketplace_products](#test-324) -- [Test 325: get_azure_capacity](#test-325) -- [Test 326: get_azure_capacity](#test-326) +- [Test 312: browse_azure_marketplace_products](#test-312) +- [Test 313: browse_azure_marketplace_products](#test-313) +- [Test 314: browse_azure_marketplace_products](#test-314) +- [Test 315: get_azure_capacity](#test-315) +- [Test 316: get_azure_capacity](#test-316) +- [Test 317: get_azure_messaging_service_details](#test-317) +- [Test 318: get_azure_messaging_service_details](#test-318) +- [Test 319: get_azure_messaging_service_details](#test-319) +- [Test 320: get_azure_messaging_service_details](#test-320) +- [Test 321: get_azure_messaging_service_details](#test-321) +- [Test 322: get_azure_messaging_service_details](#test-322) +- [Test 323: get_azure_messaging_service_details](#test-323) +- [Test 324: get_azure_messaging_service_details](#test-324) +- [Test 325: get_azure_messaging_service_details](#test-325) +- [Test 326: get_azure_messaging_service_details](#test-326) - [Test 327: get_azure_messaging_service_details](#test-327) - [Test 328: get_azure_messaging_service_details](#test-328) - [Test 329: get_azure_messaging_service_details](#test-329) @@ -349,54 +349,54 @@ - [Test 334: get_azure_messaging_service_details](#test-334) - [Test 335: get_azure_messaging_service_details](#test-335) - [Test 336: get_azure_messaging_service_details](#test-336) -- [Test 337: get_azure_messaging_service_details](#test-337) -- [Test 338: get_azure_messaging_service_details](#test-338) -- [Test 339: get_azure_messaging_service_details](#test-339) -- [Test 340: get_azure_messaging_service_details](#test-340) -- [Test 341: get_azure_messaging_service_details](#test-341) -- [Test 342: get_azure_messaging_service_details](#test-342) -- [Test 343: get_azure_messaging_service_details](#test-343) -- [Test 344: get_azure_messaging_service_details](#test-344) -- [Test 345: get_azure_messaging_service_details](#test-345) -- [Test 346: get_azure_messaging_service_details](#test-346) -- [Test 347: edit_azure_data_analytics_resources](#test-347) -- [Test 348: edit_azure_data_analytics_resources](#test-348) -- [Test 349: edit_azure_data_analytics_resources](#test-349) -- [Test 350: edit_azure_data_analytics_resources](#test-350) -- [Test 351: edit_azure_data_analytics_resources](#test-351) -- [Test 352: edit_azure_data_analytics_resources](#test-352) -- [Test 353: edit_azure_data_analytics_resources](#test-353) -- [Test 354: edit_azure_data_analytics_resources](#test-354) -- [Test 355: edit_azure_data_analytics_resources](#test-355) -- [Test 356: publish_azure_eventgrid_events](#test-356) -- [Test 357: publish_azure_eventgrid_events](#test-357) -- [Test 358: publish_azure_eventgrid_events](#test-358) +- [Test 337: edit_azure_data_analytics_resources](#test-337) +- [Test 338: edit_azure_data_analytics_resources](#test-338) +- [Test 339: edit_azure_data_analytics_resources](#test-339) +- [Test 340: edit_azure_data_analytics_resources](#test-340) +- [Test 341: edit_azure_data_analytics_resources](#test-341) +- [Test 342: edit_azure_data_analytics_resources](#test-342) +- [Test 343: edit_azure_data_analytics_resources](#test-343) +- [Test 344: edit_azure_data_analytics_resources](#test-344) +- [Test 345: edit_azure_data_analytics_resources](#test-345) +- [Test 346: publish_azure_eventgrid_events](#test-346) +- [Test 347: publish_azure_eventgrid_events](#test-347) +- [Test 348: publish_azure_eventgrid_events](#test-348) +- [Test 349: get_azure_data_explorer_kusto_details](#test-349) +- [Test 350: get_azure_data_explorer_kusto_details](#test-350) +- [Test 351: get_azure_data_explorer_kusto_details](#test-351) +- [Test 352: get_azure_data_explorer_kusto_details](#test-352) +- [Test 353: get_azure_data_explorer_kusto_details](#test-353) +- [Test 354: get_azure_data_explorer_kusto_details](#test-354) +- [Test 355: get_azure_data_explorer_kusto_details](#test-355) +- [Test 356: get_azure_data_explorer_kusto_details](#test-356) +- [Test 357: get_azure_data_explorer_kusto_details](#test-357) +- [Test 358: get_azure_data_explorer_kusto_details](#test-358) - [Test 359: get_azure_data_explorer_kusto_details](#test-359) -- [Test 360: get_azure_data_explorer_kusto_details](#test-360) -- [Test 361: get_azure_data_explorer_kusto_details](#test-361) -- [Test 362: get_azure_data_explorer_kusto_details](#test-362) -- [Test 363: get_azure_data_explorer_kusto_details](#test-363) -- [Test 364: get_azure_data_explorer_kusto_details](#test-364) -- [Test 365: get_azure_data_explorer_kusto_details](#test-365) -- [Test 366: get_azure_data_explorer_kusto_details](#test-366) -- [Test 367: get_azure_data_explorer_kusto_details](#test-367) -- [Test 368: get_azure_data_explorer_kusto_details](#test-368) -- [Test 369: get_azure_data_explorer_kusto_details](#test-369) -- [Test 370: create_azure_database_admin_configurations](#test-370) -- [Test 371: create_azure_database_admin_configurations](#test-371) -- [Test 372: create_azure_database_admin_configurations](#test-372) -- [Test 373: delete_azure_database_admin_configurations](#test-373) -- [Test 374: delete_azure_database_admin_configurations](#test-374) -- [Test 375: delete_azure_database_admin_configurations](#test-375) -- [Test 376: get_azure_database_admin_configuration_details](#test-376) -- [Test 377: get_azure_database_admin_configuration_details](#test-377) -- [Test 378: get_azure_database_admin_configuration_details](#test-378) -- [Test 379: get_azure_database_admin_configuration_details](#test-379) -- [Test 380: get_azure_database_admin_configuration_details](#test-380) -- [Test 381: get_azure_database_admin_configuration_details](#test-381) -- [Test 382: get_azure_database_admin_configuration_details](#test-382) -- [Test 383: get_azure_database_admin_configuration_details](#test-383) -- [Test 384: get_azure_database_admin_configuration_details](#test-384) +- [Test 360: create_azure_database_admin_configurations](#test-360) +- [Test 361: create_azure_database_admin_configurations](#test-361) +- [Test 362: create_azure_database_admin_configurations](#test-362) +- [Test 363: delete_azure_database_admin_configurations](#test-363) +- [Test 364: delete_azure_database_admin_configurations](#test-364) +- [Test 365: delete_azure_database_admin_configurations](#test-365) +- [Test 366: get_azure_database_admin_configuration_details](#test-366) +- [Test 367: get_azure_database_admin_configuration_details](#test-367) +- [Test 368: get_azure_database_admin_configuration_details](#test-368) +- [Test 369: get_azure_database_admin_configuration_details](#test-369) +- [Test 370: get_azure_database_admin_configuration_details](#test-370) +- [Test 371: get_azure_database_admin_configuration_details](#test-371) +- [Test 372: get_azure_database_admin_configuration_details](#test-372) +- [Test 373: get_azure_database_admin_configuration_details](#test-373) +- [Test 374: get_azure_database_admin_configuration_details](#test-374) +- [Test 375: get_azure_container_details](#test-375) +- [Test 376: get_azure_container_details](#test-376) +- [Test 377: get_azure_container_details](#test-377) +- [Test 378: get_azure_container_details](#test-378) +- [Test 379: get_azure_container_details](#test-379) +- [Test 380: get_azure_container_details](#test-380) +- [Test 381: get_azure_container_details](#test-381) +- [Test 382: get_azure_container_details](#test-382) +- [Test 383: get_azure_container_details](#test-383) +- [Test 384: get_azure_container_details](#test-384) - [Test 385: get_azure_container_details](#test-385) - [Test 386: get_azure_container_details](#test-386) - [Test 387: get_azure_container_details](#test-387) @@ -409,58 +409,48 @@ - [Test 394: get_azure_container_details](#test-394) - [Test 395: get_azure_container_details](#test-395) - [Test 396: get_azure_container_details](#test-396) -- [Test 397: get_azure_container_details](#test-397) -- [Test 398: get_azure_container_details](#test-398) -- [Test 399: get_azure_container_details](#test-399) -- [Test 400: get_azure_container_details](#test-400) -- [Test 401: get_azure_container_details](#test-401) -- [Test 402: get_azure_container_details](#test-402) -- [Test 403: get_azure_container_details](#test-403) -- [Test 404: get_azure_container_details](#test-404) -- [Test 405: get_azure_container_details](#test-405) -- [Test 406: get_azure_container_details](#test-406) -- [Test 407: get_azure_virtual_desktop_details](#test-407) -- [Test 408: get_azure_virtual_desktop_details](#test-408) -- [Test 409: get_azure_virtual_desktop_details](#test-409) -- [Test 410: get_azure_signalr_details](#test-410) -- [Test 411: get_azure_signalr_details](#test-411) -- [Test 412: get_azure_signalr_details](#test-412) -- [Test 413: get_azure_signalr_details](#test-413) -- [Test 414: get_azure_signalr_details](#test-414) -- [Test 415: get_azure_signalr_details](#test-415) -- [Test 416: get_azure_confidential_ledger_entries](#test-416) -- [Test 417: get_azure_confidential_ledger_entries](#test-417) -- [Test 418: append_azure_confidential_ledger_entries](#test-418) -- [Test 419: append_azure_confidential_ledger_entries](#test-419) -- [Test 420: append_azure_confidential_ledger_entries](#test-420) -- [Test 421: append_azure_confidential_ledger_entries](#test-421) -- [Test 422: append_azure_confidential_ledger_entries](#test-422) +- [Test 397: get_azure_virtual_desktop_details](#test-397) +- [Test 398: get_azure_virtual_desktop_details](#test-398) +- [Test 399: get_azure_virtual_desktop_details](#test-399) +- [Test 400: get_azure_signalr_details](#test-400) +- [Test 401: get_azure_signalr_details](#test-401) +- [Test 402: get_azure_signalr_details](#test-402) +- [Test 403: get_azure_signalr_details](#test-403) +- [Test 404: get_azure_signalr_details](#test-404) +- [Test 405: get_azure_signalr_details](#test-405) +- [Test 406: get_azure_confidential_ledger_entries](#test-406) +- [Test 407: get_azure_confidential_ledger_entries](#test-407) +- [Test 408: append_azure_confidential_ledger_entries](#test-408) +- [Test 409: append_azure_confidential_ledger_entries](#test-409) +- [Test 410: append_azure_confidential_ledger_entries](#test-410) +- [Test 411: append_azure_confidential_ledger_entries](#test-411) +- [Test 412: append_azure_confidential_ledger_entries](#test-412) +- [Test 413: send_azure_communication_messages](#test-413) +- [Test 414: send_azure_communication_messages](#test-414) +- [Test 415: send_azure_communication_messages](#test-415) +- [Test 416: send_azure_communication_messages](#test-416) +- [Test 417: send_azure_communication_messages](#test-417) +- [Test 418: send_azure_communication_messages](#test-418) +- [Test 419: send_azure_communication_messages](#test-419) +- [Test 420: send_azure_communication_messages](#test-420) +- [Test 421: send_azure_communication_messages](#test-421) +- [Test 422: send_azure_communication_messages](#test-422) - [Test 423: send_azure_communication_messages](#test-423) - [Test 424: send_azure_communication_messages](#test-424) - [Test 425: send_azure_communication_messages](#test-425) - [Test 426: send_azure_communication_messages](#test-426) - [Test 427: send_azure_communication_messages](#test-427) - [Test 428: send_azure_communication_messages](#test-428) -- [Test 429: send_azure_communication_messages](#test-429) -- [Test 430: send_azure_communication_messages](#test-430) -- [Test 431: send_azure_communication_messages](#test-431) -- [Test 432: send_azure_communication_messages](#test-432) -- [Test 433: send_azure_communication_messages](#test-433) -- [Test 434: send_azure_communication_messages](#test-434) -- [Test 435: send_azure_communication_messages](#test-435) -- [Test 436: send_azure_communication_messages](#test-436) -- [Test 437: send_azure_communication_messages](#test-437) -- [Test 438: send_azure_communication_messages](#test-438) -- [Test 439: recognize_speech_from_audio](#test-439) -- [Test 440: recognize_speech_from_audio](#test-440) -- [Test 441: recognize_speech_from_audio](#test-441) -- [Test 442: recognize_speech_from_audio](#test-442) -- [Test 443: recognize_speech_from_audio](#test-443) -- [Test 444: recognize_speech_from_audio](#test-444) -- [Test 445: recognize_speech_from_audio](#test-445) -- [Test 446: recognize_speech_from_audio](#test-446) -- [Test 447: recognize_speech_from_audio](#test-447) -- [Test 448: recognize_speech_from_audio](#test-448) +- [Test 429: recognize_speech_from_audio](#test-429) +- [Test 430: recognize_speech_from_audio](#test-430) +- [Test 431: recognize_speech_from_audio](#test-431) +- [Test 432: recognize_speech_from_audio](#test-432) +- [Test 433: recognize_speech_from_audio](#test-433) +- [Test 434: recognize_speech_from_audio](#test-434) +- [Test 435: recognize_speech_from_audio](#test-435) +- [Test 436: recognize_speech_from_audio](#test-436) +- [Test 437: recognize_speech_from_audio](#test-437) +- [Test 438: recognize_speech_from_audio](#test-438) --- @@ -475,9 +465,9 @@ |------|-------|------|--------| | 1 | 0.638889 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.420089 | `get_azure_security_configurations` | ❌ | -| 3 | 0.401270 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.401415 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.384567 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.366619 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.375987 | `get_azure_capacity` | ❌ | --- @@ -493,7 +483,7 @@ | 1 | 0.415793 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.328705 | `get_azure_security_configurations` | ❌ | | 3 | 0.317407 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.312982 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.313066 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.285674 | `get_azure_storage_details` | ❌ | --- @@ -509,9 +499,9 @@ |------|-------|------|--------| | 1 | 0.549609 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.418812 | `get_azure_security_configurations` | ❌ | -| 3 | 0.370016 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.364712 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.358284 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.370452 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.366696 | `get_azure_capacity` | ❌ | +| 5 | 0.364712 | `get_azure_load_testing_details` | ❌ | --- @@ -524,11 +514,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.347487 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.278204 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.242561 | `get_azure_security_configurations` | ❌ | -| 4 | 0.238716 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.189112 | `get_azure_container_details` | ❌ | +| 1 | 0.347561 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | +| 2 | 0.278264 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.242552 | `get_azure_security_configurations` | ❌ | +| 4 | 0.238929 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.189155 | `get_azure_container_details` | ❌ | --- @@ -543,9 +533,9 @@ |------|-------|------|--------| | 1 | 0.671045 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.437174 | `get_azure_security_configurations` | ❌ | -| 3 | 0.422991 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.381286 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.379037 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.423039 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.407266 | `get_azure_capacity` | ❌ | +| 5 | 0.381286 | `get_azure_load_testing_details` | ❌ | --- @@ -560,9 +550,9 @@ |------|-------|------|--------| | 1 | 0.322378 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | | 2 | 0.246134 | `browse_azure_marketplace_products` | ❌ | -| 3 | 0.233990 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.210046 | `get_azure_signalr_details` | ❌ | -| 5 | 0.201196 | `get_azure_resource_and_app_health_status` | ❌ | +| 3 | 0.234029 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.214599 | `get_azure_capacity` | ❌ | +| 5 | 0.210046 | `get_azure_signalr_details` | ❌ | --- @@ -576,10 +566,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.380158 | `get_azure_subscriptions_and_resource_groups` | ✅ **EXPECTED** | -| 2 | 0.303601 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.303497 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.274746 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.235632 | `get_azure_resource_and_app_health_status` | ❌ | -| 5 | 0.227908 | `get_azure_container_details` | ❌ | +| 4 | 0.248135 | `get_azure_capacity` | ❌ | +| 5 | 0.239796 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -643,11 +633,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.606747 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.516830 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.498783 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.468835 | `get_azure_signalr_details` | ❌ | -| 5 | 0.428887 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.606770 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.516861 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.498775 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.468858 | `get_azure_signalr_details` | ❌ | +| 5 | 0.429110 | `get_azure_messaging_service_details` | ❌ | --- @@ -663,7 +653,7 @@ | 1 | 0.558485 | `get_azure_app_resource_details` | ✅ **EXPECTED** | | 2 | 0.427410 | `get_azure_security_configurations` | ❌ | | 3 | 0.421965 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.420496 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.420740 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.409842 | `get_azure_resource_and_app_health_status` | ❌ | --- @@ -698,7 +688,7 @@ | 2 | 0.514721 | `get_azure_app_config_settings` | ❌ | | 3 | 0.434995 | `get_azure_signalr_details` | ❌ | | 4 | 0.430445 | `deploy_azure_resources_and_applications` | ❌ | -| 5 | 0.417163 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.417338 | `get_azure_messaging_service_details` | ❌ | --- @@ -731,7 +721,7 @@ | 1 | 0.650735 | `get_azure_app_resource_details` | ✅ **EXPECTED** | | 2 | 0.570557 | `get_azure_app_config_settings` | ❌ | | 3 | 0.467885 | `get_azure_signalr_details` | ❌ | -| 4 | 0.426620 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.427326 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.394452 | `deploy_azure_resources_and_applications` | ❌ | --- @@ -748,7 +738,7 @@ | 1 | 0.534810 | `get_azure_app_resource_details` | ✅ **EXPECTED** | | 2 | 0.433102 | `get_azure_app_config_settings` | ❌ | | 3 | 0.428962 | `deploy_azure_resources_and_applications` | ❌ | -| 4 | 0.390991 | `get_azure_capacity` | ❌ | +| 4 | 0.415746 | `get_azure_capacity` | ❌ | | 5 | 0.390919 | `get_azure_best_practices` | ❌ | --- @@ -766,7 +756,7 @@ | 2 | 0.320318 | `get_azure_resource_and_app_health_status` | ❌ | | 3 | 0.305243 | `deploy_azure_resources_and_applications` | ❌ | | 4 | 0.269663 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.263333 | `execute_azure_deployments` | ❌ | +| 5 | 0.259482 | `generate_azure_cli_commands` | ❌ | --- @@ -779,11 +769,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.547155 | `get_azure_app_resource_details` | ✅ **EXPECTED** | -| 2 | 0.435715 | `get_azure_resource_and_app_health_status` | ❌ | -| 3 | 0.419457 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.388241 | `deploy_azure_resources_and_applications` | ❌ | -| 5 | 0.385296 | `get_azure_signalr_details` | ❌ | +| 1 | 0.547132 | `get_azure_app_resource_details` | ✅ **EXPECTED** | +| 2 | 0.435720 | `get_azure_resource_and_app_health_status` | ❌ | +| 3 | 0.419467 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.388259 | `deploy_azure_resources_and_applications` | ❌ | +| 5 | 0.385260 | `get_azure_signalr_details` | ❌ | --- @@ -796,7 +786,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.645561 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.645526 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.460200 | `get_azure_databases_details` | ❌ | | 3 | 0.441681 | `rename_azure_sql_databases` | ❌ | | 4 | 0.441557 | `edit_azure_databases` | ❌ | @@ -813,7 +803,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.604845 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.604807 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.399204 | `edit_azure_databases` | ❌ | | 3 | 0.374759 | `create_azure_sql_databases_and_servers` | ❌ | | 4 | 0.371454 | `rename_azure_sql_databases` | ❌ | @@ -830,7 +820,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.662973 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.662946 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.510959 | `edit_azure_databases` | ❌ | | 3 | 0.461314 | `get_azure_databases_details` | ❌ | | 4 | 0.409706 | `rename_azure_sql_databases` | ❌ | @@ -847,7 +837,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607043 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.607029 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.488067 | `edit_azure_databases` | ❌ | | 3 | 0.423578 | `get_azure_databases_details` | ❌ | | 4 | 0.352923 | `create_azure_sql_databases_and_servers` | ❌ | @@ -864,11 +854,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612125 | `add_azure_app_service_database` | ✅ **EXPECTED** | -| 2 | 0.488271 | `rename_azure_sql_databases` | ❌ | -| 3 | 0.455473 | `create_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.445528 | `edit_azure_databases` | ❌ | -| 5 | 0.436024 | `get_azure_databases_details` | ❌ | +| 1 | 0.612095 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 2 | 0.488180 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.455573 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.445556 | `edit_azure_databases` | ❌ | +| 5 | 0.436063 | `get_azure_databases_details` | ❌ | --- @@ -881,7 +871,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550393 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.550372 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.418395 | `rename_azure_sql_databases` | ❌ | | 3 | 0.398384 | `create_azure_sql_databases_and_servers` | ❌ | | 4 | 0.380948 | `edit_azure_databases` | ❌ | @@ -898,7 +888,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.637303 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.637234 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.518406 | `create_azure_sql_databases_and_servers` | ❌ | | 3 | 0.512464 | `edit_azure_databases` | ❌ | | 4 | 0.476589 | `rename_azure_sql_databases` | ❌ | @@ -915,7 +905,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.539505 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.539469 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.446249 | `rename_azure_sql_databases` | ❌ | | 3 | 0.427201 | `edit_azure_databases` | ❌ | | 4 | 0.397353 | `lock_unlock_azure_app_config_settings` | ❌ | @@ -932,7 +922,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543995 | `add_azure_app_service_database` | ✅ **EXPECTED** | +| 1 | 0.543922 | `add_azure_app_service_database` | ✅ **EXPECTED** | | 2 | 0.468293 | `edit_azure_databases` | ❌ | | 3 | 0.428196 | `rename_azure_sql_databases` | ❌ | | 4 | 0.395735 | `lock_unlock_azure_app_config_settings` | ❌ | @@ -949,7 +939,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.394526 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.394601 | `get_azure_database_admin_configuration_details` | ❌ | | 2 | 0.378321 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.346100 | `edit_azure_databases` | ❌ | | 4 | 0.321543 | `create_azure_sql_databases_and_servers` | ❌ | @@ -966,11 +956,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516088 | `get_azure_database_admin_configuration_details` | ❌ | -| 2 | 0.515522 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.369146 | `edit_azure_databases` | ❌ | -| 4 | 0.349769 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.348912 | `create_azure_sql_databases_and_servers` | ❌ | +| 1 | 0.515990 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.515480 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.369218 | `edit_azure_databases` | ❌ | +| 4 | 0.349838 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.348994 | `create_azure_sql_databases_and_servers` | ❌ | --- @@ -983,7 +973,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.478446 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.478394 | `get_azure_database_admin_configuration_details` | ❌ | | 2 | 0.449240 | `get_azure_app_config_settings` | ❌ | | 3 | 0.378667 | `edit_azure_sql_databases_and_servers` | ❌ | | 4 | 0.375171 | `get_azure_databases_details` | ✅ **EXPECTED** | @@ -1004,7 +994,7 @@ | 2 | 0.435676 | `create_azure_sql_databases_and_servers` | ❌ | | 3 | 0.424056 | `rename_azure_sql_databases` | ❌ | | 4 | 0.423838 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 5 | 0.422601 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.422579 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1021,7 +1011,7 @@ | 2 | 0.470240 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 3 | 0.450588 | `get_azure_storage_details` | ❌ | | 4 | 0.439142 | `get_azure_security_configurations` | ❌ | -| 5 | 0.419961 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.421076 | `get_azure_messaging_service_details` | ❌ | --- @@ -1034,11 +1024,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550071 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.538282 | `rename_azure_sql_databases` | ❌ | -| 3 | 0.450003 | `create_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.447551 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.446845 | `edit_azure_sql_databases_and_servers` | ❌ | +| 1 | 0.550050 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.538253 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.449947 | `create_azure_sql_databases_and_servers` | ❌ | +| 4 | 0.447448 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.446772 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1055,7 +1045,7 @@ | 2 | 0.353887 | `edit_azure_databases` | ❌ | | 3 | 0.334742 | `rename_azure_sql_databases` | ❌ | | 4 | 0.272054 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.255201 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.255147 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1070,7 +1060,7 @@ |------|-------|------|--------| | 1 | 0.496395 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.418769 | `edit_azure_databases` | ❌ | -| 3 | 0.335256 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.335259 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.322093 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 5 | 0.316396 | `edit_azure_sql_databases_and_servers` | ❌ | @@ -1088,7 +1078,7 @@ | 1 | 0.411217 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.321958 | `edit_azure_databases` | ❌ | | 3 | 0.293602 | `rename_azure_sql_databases` | ❌ | -| 4 | 0.243270 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.243209 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.234938 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1104,7 +1094,7 @@ |------|-------|------|--------| | 1 | 0.455154 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.397754 | `edit_azure_databases` | ❌ | -| 3 | 0.341460 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.341439 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.319877 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 5 | 0.298417 | `create_azure_sql_databases_and_servers` | ❌ | @@ -1123,7 +1113,7 @@ | 2 | 0.319283 | `edit_azure_databases` | ❌ | | 3 | 0.284157 | `rename_azure_sql_databases` | ❌ | | 4 | 0.253452 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.224069 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.224061 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -1157,7 +1147,7 @@ | 2 | 0.410568 | `get_azure_storage_details` | ❌ | | 3 | 0.393277 | `get_azure_container_details` | ❌ | | 4 | 0.379221 | `create_azure_storage` | ❌ | -| 5 | 0.360666 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.331050 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -1170,11 +1160,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521563 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.403940 | `rename_azure_sql_databases` | ❌ | -| 3 | 0.372599 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.356285 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.353188 | `get_azure_storage_details` | ❌ | +| 1 | 0.521537 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.403937 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.372579 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.353114 | `get_azure_storage_details` | ❌ | +| 5 | 0.340541 | `get_azure_security_configurations` | ❌ | --- @@ -1204,11 +1194,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.357640 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.282367 | `edit_azure_databases` | ❌ | -| 3 | 0.276725 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 4 | 0.230752 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.228384 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.357501 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.282346 | `edit_azure_databases` | ❌ | +| 3 | 0.276540 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.230739 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.228187 | `get_azure_ai_resources_details` | ❌ | --- @@ -1221,7 +1211,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.558434 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.558366 | `get_azure_database_admin_configuration_details` | ❌ | | 2 | 0.530329 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.486536 | `get_azure_app_config_settings` | ❌ | | 4 | 0.468156 | `edit_azure_databases` | ❌ | @@ -1239,10 +1229,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.459302 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.424009 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.424000 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.422684 | `get_azure_databases_details` | ✅ **EXPECTED** | | 4 | 0.409588 | `create_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.386413 | `get_azure_capacity` | ❌ | +| 5 | 0.396496 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -1257,7 +1247,7 @@ |------|-------|------|--------| | 1 | 0.318903 | `edit_azure_databases` | ❌ | | 2 | 0.251389 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 3 | 0.215865 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.215877 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.168314 | `get_azure_signalr_details` | ❌ | | 5 | 0.165104 | `create_azure_database_admin_configurations` | ❌ | @@ -1292,7 +1282,7 @@ | 1 | 0.404968 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.357185 | `edit_azure_databases` | ❌ | | 3 | 0.274336 | `create_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.270829 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.270884 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.264802 | `edit_azure_sql_databases_and_servers` | ❌ | --- @@ -1308,7 +1298,7 @@ |------|-------|------|--------| | 1 | 0.380591 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.354214 | `edit_azure_databases` | ❌ | -| 3 | 0.274988 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.274990 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.253387 | `create_azure_sql_databases_and_servers` | ❌ | | 5 | 0.240477 | `delete_azure_database_admin_configurations` | ❌ | @@ -1325,7 +1315,7 @@ |------|-------|------|--------| | 1 | 0.424259 | `edit_azure_databases` | ❌ | | 2 | 0.345881 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 3 | 0.319072 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.318976 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.282061 | `get_azure_app_config_settings` | ❌ | | 5 | 0.252978 | `edit_azure_sql_databases_and_servers` | ❌ | @@ -1341,7 +1331,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.396395 | `edit_azure_databases` | ❌ | -| 2 | 0.305067 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.305000 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.302014 | `get_azure_databases_details` | ✅ **EXPECTED** | | 4 | 0.250062 | `get_azure_app_config_settings` | ❌ | | 5 | 0.214600 | `update_azure_managed_lustre_filesystems` | ❌ | @@ -1361,7 +1351,7 @@ | 2 | 0.380264 | `get_azure_storage_details` | ❌ | | 3 | 0.377338 | `get_azure_container_details` | ❌ | | 4 | 0.351073 | `create_azure_storage` | ❌ | -| 5 | 0.327822 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.325368 | `rename_azure_sql_databases` | ❌ | --- @@ -1378,7 +1368,7 @@ | 2 | 0.481986 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.458807 | `get_azure_storage_details` | ❌ | | 4 | 0.444953 | `get_azure_security_configurations` | ❌ | -| 5 | 0.430545 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.431831 | `get_azure_messaging_service_details` | ❌ | --- @@ -1395,7 +1385,7 @@ | 2 | 0.380341 | `rename_azure_sql_databases` | ❌ | | 3 | 0.368375 | `get_azure_data_explorer_kusto_details` | ❌ | | 4 | 0.340381 | `get_azure_storage_details` | ❌ | -| 5 | 0.336644 | `add_azure_app_service_database` | ❌ | +| 5 | 0.336643 | `add_azure_app_service_database` | ❌ | --- @@ -1408,11 +1398,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549664 | `get_azure_database_admin_configuration_details` | ❌ | -| 2 | 0.500276 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.549599 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.500933 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.493713 | `get_azure_signalr_details` | ❌ | -| 4 | 0.489035 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.464348 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.490565 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.489035 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- @@ -1426,7 +1416,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.430729 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.415590 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.415495 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.354815 | `edit_azure_databases` | ❌ | | 4 | 0.343500 | `get_azure_data_explorer_kusto_details` | ❌ | | 5 | 0.336660 | `rename_azure_sql_databases` | ❌ | @@ -1479,7 +1469,7 @@ | 1 | 0.511745 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.461477 | `edit_azure_databases` | ❌ | | 3 | 0.365358 | `create_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.364350 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.364272 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.353379 | `browse_azure_marketplace_products` | ❌ | --- @@ -1497,7 +1487,7 @@ | 2 | 0.332482 | `edit_azure_databases` | ❌ | | 3 | 0.277752 | `rename_azure_sql_databases` | ❌ | | 4 | 0.247838 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.245580 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.245490 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1512,7 +1502,7 @@ |------|-------|------|--------| | 1 | 0.469057 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.433648 | `edit_azure_databases` | ❌ | -| 3 | 0.360426 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.360364 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.343503 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 5 | 0.339133 | `browse_azure_marketplace_products` | ❌ | @@ -1527,11 +1517,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.382863 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 1 | 0.383189 | `get_azure_databases_details` | ✅ **EXPECTED** | | 2 | 0.314466 | `edit_azure_databases` | ❌ | -| 3 | 0.225336 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.224832 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.221369 | `get_azure_best_practices` | ❌ | +| 3 | 0.225610 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.224830 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.221419 | `get_azure_best_practices` | ❌ | --- @@ -1544,11 +1534,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.343844 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.284626 | `edit_azure_databases` | ❌ | -| 3 | 0.214230 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.201567 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.199864 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.344385 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.284782 | `edit_azure_databases` | ❌ | +| 3 | 0.214431 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.201470 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.200118 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1565,7 +1555,7 @@ | 2 | 0.335190 | `edit_azure_databases` | ❌ | | 3 | 0.262481 | `rename_azure_sql_databases` | ❌ | | 4 | 0.256053 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.249102 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.248928 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1578,11 +1568,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.375836 | `get_azure_databases_details` | ✅ **EXPECTED** | -| 2 | 0.311404 | `edit_azure_databases` | ❌ | -| 3 | 0.230790 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.226153 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.224314 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.375595 | `get_azure_databases_details` | ✅ **EXPECTED** | +| 2 | 0.310370 | `edit_azure_databases` | ❌ | +| 3 | 0.230608 | `get_azure_data_explorer_kusto_details` | ❌ | +| 4 | 0.225953 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.223967 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1599,7 +1589,7 @@ | 2 | 0.238900 | `get_azure_databases_details` | ✅ **EXPECTED** | | 3 | 0.202957 | `delete_azure_database_admin_configurations` | ❌ | | 4 | 0.202122 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.192494 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.192344 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -1629,11 +1619,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520385 | `rename_azure_sql_databases` | ❌ | -| 2 | 0.513431 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | -| 3 | 0.384182 | `add_azure_app_service_database` | ❌ | -| 4 | 0.381023 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.376654 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.520646 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.513516 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.384104 | `add_azure_app_service_database` | ❌ | +| 4 | 0.381160 | `edit_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.376755 | `create_azure_database_admin_configurations` | ❌ | --- @@ -1667,7 +1657,7 @@ | 2 | 0.448354 | `rename_azure_sql_databases` | ❌ | | 3 | 0.429547 | `edit_azure_sql_databases_and_servers` | ❌ | | 4 | 0.414021 | `edit_azure_databases` | ❌ | -| 5 | 0.390212 | `add_azure_app_service_database` | ❌ | +| 5 | 0.390204 | `add_azure_app_service_database` | ❌ | --- @@ -1682,7 +1672,7 @@ |------|-------|------|--------| | 1 | 0.502365 | `create_azure_sql_databases_and_servers` | ✅ **EXPECTED** | | 2 | 0.468619 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.424715 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.424808 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.411480 | `edit_azure_sql_databases_and_servers` | ❌ | | 5 | 0.411012 | `rename_azure_sql_databases` | ❌ | @@ -1701,7 +1691,7 @@ | 2 | 0.485689 | `rename_azure_sql_databases` | ❌ | | 3 | 0.442793 | `create_azure_database_admin_configurations` | ❌ | | 4 | 0.407416 | `edit_azure_databases` | ❌ | -| 5 | 0.391782 | `add_azure_app_service_database` | ❌ | +| 5 | 0.391741 | `add_azure_app_service_database` | ❌ | --- @@ -1714,11 +1704,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.794416 | `rename_azure_sql_databases` | ✅ **EXPECTED** | -| 2 | 0.541638 | `edit_azure_sql_databases_and_servers` | ❌ | -| 3 | 0.505153 | `edit_azure_databases` | ❌ | -| 4 | 0.475451 | `create_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.437195 | `create_azure_database_admin_configurations` | ❌ | +| 1 | 0.794632 | `rename_azure_sql_databases` | ✅ **EXPECTED** | +| 2 | 0.541921 | `edit_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.505707 | `edit_azure_databases` | ❌ | +| 4 | 0.475553 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.437481 | `create_azure_database_admin_configurations` | ❌ | --- @@ -1765,11 +1755,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530935 | `delete_azure_database_admin_configurations` | ❌ | -| 2 | 0.524563 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | -| 3 | 0.470808 | `rename_azure_sql_databases` | ❌ | -| 4 | 0.453765 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.417285 | `edit_azure_data_analytics_resources` | ❌ | +| 1 | 0.530932 | `delete_azure_database_admin_configurations` | ❌ | +| 2 | 0.524407 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 3 | 0.470752 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.453689 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.417277 | `edit_azure_data_analytics_resources` | ❌ | --- @@ -1850,11 +1840,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.447788 | `rename_azure_sql_databases` | ❌ | -| 2 | 0.446405 | `edit_azure_databases` | ❌ | -| 3 | 0.444369 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | -| 4 | 0.434542 | `create_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.348544 | `get_azure_databases_details` | ❌ | +| 1 | 0.447543 | `rename_azure_sql_databases` | ❌ | +| 2 | 0.446459 | `edit_azure_databases` | ❌ | +| 3 | 0.444263 | `edit_azure_sql_databases_and_servers` | ✅ **EXPECTED** | +| 4 | 0.434377 | `create_azure_sql_databases_and_servers` | ❌ | +| 5 | 0.348517 | `get_azure_databases_details` | ❌ | --- @@ -1887,7 +1877,7 @@ | 1 | 0.340843 | `edit_azure_databases` | ✅ **EXPECTED** | | 2 | 0.250565 | `create_azure_database_admin_configurations` | ❌ | | 3 | 0.234603 | `get_azure_databases_details` | ❌ | -| 4 | 0.220308 | `add_azure_app_service_database` | ❌ | +| 4 | 0.220379 | `add_azure_app_service_database` | ❌ | | 5 | 0.212755 | `delete_azure_database_admin_configurations` | ❌ | --- @@ -1935,11 +1925,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510833 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.472122 | `create_azure_monitor_webtests` | ❌ | -| 3 | 0.459627 | `update_azure_monitor_webtests` | ❌ | -| 4 | 0.426727 | `get_azure_capacity` | ❌ | -| 5 | 0.381483 | `get_azure_load_testing_details` | ❌ | +| 1 | 0.510827 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.472245 | `create_azure_monitor_webtests` | ❌ | +| 3 | 0.459723 | `update_azure_monitor_webtests` | ❌ | +| 4 | 0.438668 | `get_azure_capacity` | ❌ | +| 5 | 0.381326 | `get_azure_load_testing_details` | ❌ | --- @@ -1952,11 +1942,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.345032 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.345277 | `get_azure_messaging_service_details` | ❌ | | 2 | 0.302173 | `get_azure_storage_details` | ❌ | -| 3 | 0.285925 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 4 | 0.272938 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.271450 | `get_azure_capacity` | ❌ | +| 3 | 0.289476 | `get_azure_capacity` | ❌ | +| 4 | 0.285925 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 5 | 0.276066 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -1969,11 +1959,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.314111 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.257445 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.253789 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.244128 | `get_azure_load_testing_details` | ❌ | -| 5 | 0.239832 | `get_azure_signalr_details` | ❌ | +| 1 | 0.314030 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.257651 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.253620 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.244006 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.239936 | `get_azure_signalr_details` | ❌ | --- @@ -1986,11 +1976,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.374257 | `get_azure_capacity` | ❌ | +| 1 | 0.380856 | `get_azure_capacity` | ❌ | | 2 | 0.338439 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 3 | 0.336667 | `get_azure_storage_details` | ❌ | | 4 | 0.325422 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.305299 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.312349 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -2005,8 +1995,8 @@ |------|-------|------|--------| | 1 | 0.486070 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.406567 | `create_azure_monitor_webtests` | ❌ | -| 3 | 0.389526 | `get_azure_capacity` | ❌ | -| 4 | 0.385006 | `update_azure_monitor_webtests` | ❌ | +| 3 | 0.405853 | `get_azure_capacity` | ❌ | +| 4 | 0.384944 | `update_azure_monitor_webtests` | ❌ | | 5 | 0.375039 | `create_azure_load_testing` | ❌ | --- @@ -2021,10 +2011,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.508303 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.487675 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.487631 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.384016 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.380883 | `get_azure_signalr_details` | ❌ | -| 5 | 0.379399 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.379623 | `publish_azure_eventgrid_events` | ❌ | --- @@ -2058,7 +2048,7 @@ | 2 | 0.449685 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 3 | 0.445863 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.438518 | `get_azure_security_configurations` | ❌ | -| 5 | 0.423144 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.432854 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -2091,8 +2081,8 @@ | 1 | 0.400161 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.356965 | `get_azure_load_testing_details` | ❌ | | 3 | 0.349385 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.321621 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.318657 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.343187 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.318702 | `get_azure_messaging_service_details` | ❌ | --- @@ -2106,9 +2096,9 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.514546 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.487976 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.488013 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.381633 | `get_azure_signalr_details` | ❌ | -| 4 | 0.372535 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.372903 | `publish_azure_eventgrid_events` | ❌ | | 5 | 0.371822 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- @@ -2122,11 +2112,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.429580 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.418506 | `get_azure_workbooks_details` | ❌ | -| 3 | 0.398756 | `create_azure_workbooks` | ❌ | -| 4 | 0.393661 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.376718 | `get_azure_databases_details` | ❌ | +| 1 | 0.429517 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.418441 | `get_azure_workbooks_details` | ❌ | +| 3 | 0.398664 | `create_azure_workbooks` | ❌ | +| 4 | 0.393598 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.376632 | `get_azure_databases_details` | ❌ | --- @@ -2140,9 +2130,9 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.525913 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.482171 | `get_azure_storage_details` | ❌ | -| 3 | 0.479549 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.478110 | `get_azure_capacity` | ❌ | +| 2 | 0.514752 | `get_azure_capacity` | ❌ | +| 3 | 0.482171 | `get_azure_storage_details` | ❌ | +| 4 | 0.480031 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.465854 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | --- @@ -2159,7 +2149,7 @@ | 1 | 0.474724 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.430555 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.423122 | `get_azure_best_practices` | ❌ | -| 4 | 0.407439 | `update_azure_monitor_webtests` | ❌ | +| 4 | 0.407367 | `update_azure_monitor_webtests` | ❌ | | 5 | 0.368902 | `create_azure_monitor_webtests` | ❌ | --- @@ -2176,8 +2166,8 @@ | 1 | 0.531640 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.453872 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.439990 | `get_azure_ai_resources_details` | ❌ | -| 4 | 0.416257 | `get_azure_security_configurations` | ❌ | -| 5 | 0.416176 | `get_azure_capacity` | ❌ | +| 4 | 0.418057 | `get_azure_capacity` | ❌ | +| 5 | 0.416257 | `get_azure_security_configurations` | ❌ | --- @@ -2193,8 +2183,8 @@ | 1 | 0.340942 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.293858 | `get_azure_load_testing_details` | ❌ | | 3 | 0.279505 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.269796 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.260043 | `get_azure_capacity` | ❌ | +| 4 | 0.269020 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.267492 | `get_azure_capacity` | ❌ | --- @@ -2210,7 +2200,7 @@ | 1 | 0.348670 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.247328 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.246532 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.235710 | `update_azure_monitor_webtests` | ❌ | +| 4 | 0.235685 | `update_azure_monitor_webtests` | ❌ | | 5 | 0.235128 | `get_azure_app_resource_details` | ❌ | --- @@ -2227,8 +2217,8 @@ | 1 | 0.330121 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.264069 | `get_azure_virtual_desktop_details` | ❌ | | 3 | 0.253364 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.252579 | `get_azure_capacity` | ❌ | -| 5 | 0.245564 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.252330 | `get_azure_capacity` | ❌ | +| 5 | 0.245600 | `get_azure_messaging_service_details` | ❌ | --- @@ -2242,8 +2232,8 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.558407 | `get_azure_storage_details` | ❌ | -| 2 | 0.430781 | `get_azure_capacity` | ❌ | -| 3 | 0.405616 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.447481 | `get_azure_capacity` | ❌ | +| 3 | 0.407697 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.403095 | `create_azure_storage` | ❌ | | 5 | 0.390433 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | @@ -2259,10 +2249,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.496308 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.467641 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.467743 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.424333 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.392392 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.387040 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.392324 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.390496 | `get_azure_capacity` | ❌ | --- @@ -2278,8 +2268,8 @@ | 1 | 0.508215 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.492356 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.464604 | `get_azure_best_practices` | ❌ | -| 4 | 0.421210 | `get_azure_capacity` | ❌ | -| 5 | 0.418559 | `create_azure_monitor_webtests` | ❌ | +| 4 | 0.418559 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.413326 | `get_azure_capacity` | ❌ | --- @@ -2310,10 +2300,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.472052 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.434889 | `update_azure_monitor_webtests` | ❌ | +| 2 | 0.434828 | `update_azure_monitor_webtests` | ❌ | | 3 | 0.411997 | `evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.398711 | `create_azure_monitor_webtests` | ❌ | -| 5 | 0.396968 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.396961 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -2327,10 +2317,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.478758 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.467913 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.468817 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.445424 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.435005 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.427813 | `get_azure_capacity` | ❌ | +| 4 | 0.438080 | `get_azure_capacity` | ❌ | +| 5 | 0.435005 | `get_azure_app_resource_details` | ❌ | --- @@ -2361,9 +2351,9 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.575009 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.488089 | `get_azure_capacity` | ❌ | +| 2 | 0.499791 | `get_azure_capacity` | ❌ | | 3 | 0.475264 | `get_azure_virtual_desktop_details` | ❌ | -| 4 | 0.473441 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.474203 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.471465 | `get_azure_security_configurations` | ❌ | --- @@ -2379,9 +2369,9 @@ |------|-------|------|--------| | 1 | 0.311450 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.230296 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.209276 | `get_azure_capacity` | ❌ | -| 4 | 0.201724 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.193404 | `audit_azure_resources_compliance` | ❌ | +| 3 | 0.201724 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.193404 | `audit_azure_resources_compliance` | ❌ | +| 5 | 0.190307 | `create_azure_monitor_webtests` | ❌ | --- @@ -2394,11 +2384,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.494538 | `get_azure_storage_details` | ❌ | -| 2 | 0.403753 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 3 | 0.395192 | `create_azure_storage` | ❌ | -| 4 | 0.368143 | `get_azure_capacity` | ❌ | -| 5 | 0.339448 | `get_azure_security_configurations` | ❌ | +| 1 | 0.494473 | `get_azure_storage_details` | ❌ | +| 2 | 0.403725 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 3 | 0.395150 | `create_azure_storage` | ❌ | +| 4 | 0.371528 | `get_azure_capacity` | ❌ | +| 5 | 0.339439 | `get_azure_security_configurations` | ❌ | --- @@ -2431,8 +2421,8 @@ | 1 | 0.464430 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.365333 | `get_azure_virtual_desktop_details` | ❌ | | 3 | 0.333899 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.332443 | `get_azure_capacity` | ❌ | -| 5 | 0.329728 | `get_azure_workbooks_details` | ❌ | +| 4 | 0.329728 | `get_azure_workbooks_details` | ❌ | +| 5 | 0.327688 | `get_azure_signalr_details` | ❌ | --- @@ -2464,9 +2454,9 @@ |------|-------|------|--------| | 1 | 0.398557 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.354368 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.346010 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.321110 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.316100 | `get_azure_cache_for_redis_details` | ❌ | +| 3 | 0.347808 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.346010 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.324679 | `get_azure_capacity` | ❌ | --- @@ -2516,7 +2506,7 @@ | 1 | 0.299595 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.250948 | `get_azure_app_resource_details` | ❌ | | 3 | 0.212919 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.205952 | `add_azure_app_service_database` | ❌ | +| 4 | 0.205979 | `add_azure_app_service_database` | ❌ | | 5 | 0.201566 | `get_azure_signalr_details` | ❌ | --- @@ -2530,8 +2520,8 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.421696 | `get_azure_capacity` | ❌ | -| 2 | 0.404977 | `get_azure_virtual_desktop_details` | ❌ | +| 1 | 0.404977 | `get_azure_virtual_desktop_details` | ❌ | +| 2 | 0.381848 | `get_azure_capacity` | ❌ | | 3 | 0.380088 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.377460 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 5 | 0.363367 | `get_azure_signalr_details` | ❌ | @@ -2549,9 +2539,9 @@ |------|-------|------|--------| | 1 | 0.519763 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | | 2 | 0.415402 | `create_azure_monitor_webtests` | ❌ | -| 3 | 0.411633 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.410211 | `update_azure_monitor_webtests` | ❌ | -| 5 | 0.397417 | `get_azure_capacity` | ❌ | +| 3 | 0.411354 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.410131 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.409756 | `get_azure_capacity` | ❌ | --- @@ -2565,8 +2555,8 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.505430 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.423077 | `get_azure_capacity` | ❌ | -| 3 | 0.412065 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.411932 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.408064 | `get_azure_capacity` | ❌ | | 4 | 0.389687 | `get_azure_virtual_desktop_details` | ❌ | | 5 | 0.382850 | `get_azure_subscriptions_and_resource_groups` | ❌ | @@ -2582,10 +2572,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.274285 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.256635 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.256350 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.237971 | `get_azure_signalr_details` | ❌ | -| 4 | 0.226131 | `get_azure_capacity` | ❌ | -| 5 | 0.225301 | `get_azure_container_details` | ❌ | +| 4 | 0.225301 | `get_azure_container_details` | ❌ | +| 5 | 0.224471 | `get_azure_capacity` | ❌ | --- @@ -2598,11 +2588,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.437195 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | -| 2 | 0.395986 | `get_azure_capacity` | ❌ | -| 3 | 0.371019 | `get_azure_signalr_details` | ❌ | -| 4 | 0.369677 | `create_azure_load_testing` | ❌ | -| 5 | 0.363915 | `create_azure_monitor_webtests` | ❌ | +| 1 | 0.438760 | `get_azure_resource_and_app_health_status` | ✅ **EXPECTED** | +| 2 | 0.399002 | `get_azure_capacity` | ❌ | +| 3 | 0.371290 | `get_azure_signalr_details` | ❌ | +| 4 | 0.370369 | `create_azure_load_testing` | ❌ | +| 5 | 0.363028 | `create_azure_monitor_webtests` | ❌ | --- @@ -2616,10 +2606,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.640058 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | -| 2 | 0.507616 | `execute_azure_deployments` | ❌ | -| 3 | 0.479918 | `get_azure_best_practices` | ❌ | -| 4 | 0.461377 | `create_azure_monitor_webtests` | ❌ | -| 5 | 0.454870 | `add_azure_app_service_database` | ❌ | +| 2 | 0.479918 | `get_azure_best_practices` | ❌ | +| 3 | 0.461377 | `create_azure_monitor_webtests` | ❌ | +| 4 | 0.454825 | `add_azure_app_service_database` | ❌ | +| 5 | 0.453039 | `design_azure_architecture` | ❌ | --- @@ -2632,11 +2622,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.578541 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | -| 2 | 0.505622 | `execute_azure_deployments` | ❌ | -| 3 | 0.428397 | `add_azure_app_service_database` | ❌ | -| 4 | 0.410766 | `get_azure_best_practices` | ❌ | -| 5 | 0.372703 | `generate_azure_cli_commands` | ❌ | +| 1 | 0.578477 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 2 | 0.428250 | `add_azure_app_service_database` | ❌ | +| 3 | 0.410719 | `get_azure_best_practices` | ❌ | +| 4 | 0.409389 | `deploy_azure_ai_models` | ❌ | +| 5 | 0.372647 | `generate_azure_cli_commands` | ❌ | --- @@ -2649,11 +2639,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.562465 | `execute_azure_deployments` | ❌ | -| 2 | 0.522934 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | -| 3 | 0.442200 | `get_azure_resource_and_app_health_status` | ❌ | -| 4 | 0.402454 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.392628 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.522934 | `deploy_azure_resources_and_applications` | ✅ **EXPECTED** | +| 2 | 0.442200 | `get_azure_resource_and_app_health_status` | ❌ | +| 3 | 0.402454 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.392628 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.385063 | `get_azure_container_details` | ❌ | --- @@ -2676,7 +2666,7 @@ ## Test 131 -**Expected Tool:** `execute_azure_deployments` +**Expected Tool:** `deploy_azure_ai_models` **Prompt:** Deploy a GPT4o instance on my resource ### Results @@ -2685,9 +2675,9 @@ |------|-------|------|--------| | 1 | 0.307463 | `deploy_azure_resources_and_applications` | ❌ | | 2 | 0.299302 | `create_azure_load_testing` | ❌ | -| 3 | 0.240425 | `edit_azure_databases` | ❌ | -| 4 | 0.236281 | `get_azure_best_practices` | ❌ | -| 5 | 0.234060 | `execute_azure_deployments` | ✅ **EXPECTED** | +| 3 | 0.281815 | `deploy_azure_ai_models` | ✅ **EXPECTED** | +| 4 | 0.240425 | `edit_azure_databases` | ❌ | +| 5 | 0.236281 | `get_azure_best_practices` | ❌ | --- @@ -2703,8 +2693,8 @@ | 1 | 0.549804 | `get_azure_app_config_settings` | ✅ **EXPECTED** | | 2 | 0.418698 | `lock_unlock_azure_app_config_settings` | ❌ | | 3 | 0.401422 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.388838 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.378481 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.388733 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.379923 | `get_azure_messaging_service_details` | ❌ | --- @@ -2721,7 +2711,7 @@ | 2 | 0.469735 | `lock_unlock_azure_app_config_settings` | ❌ | | 3 | 0.413315 | `edit_azure_app_config_settings` | ❌ | | 4 | 0.323275 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.310582 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.310593 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -2734,11 +2724,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.492475 | `get_azure_app_config_settings` | ✅ **EXPECTED** | -| 2 | 0.374798 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.344491 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.292927 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.275008 | `get_azure_key_vault_items` | ❌ | +| 1 | 0.492499 | `get_azure_app_config_settings` | ✅ **EXPECTED** | +| 2 | 0.375253 | `lock_unlock_azure_app_config_settings` | ❌ | +| 3 | 0.345143 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.293230 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.274244 | `get_azure_key_vault_items` | ❌ | --- @@ -2753,9 +2743,9 @@ |------|-------|------|--------| | 1 | 0.517123 | `get_azure_app_config_settings` | ✅ **EXPECTED** | | 2 | 0.397359 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.350017 | `add_azure_app_service_database` | ❌ | +| 3 | 0.350023 | `add_azure_app_service_database` | ❌ | | 4 | 0.318242 | `edit_azure_app_config_settings` | ❌ | -| 5 | 0.300071 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.299920 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -2770,9 +2760,9 @@ |------|-------|------|--------| | 1 | 0.564754 | `get_azure_app_config_settings` | ✅ **EXPECTED** | | 2 | 0.445478 | `lock_unlock_azure_app_config_settings` | ❌ | -| 3 | 0.410372 | `add_azure_app_service_database` | ❌ | +| 3 | 0.410403 | `add_azure_app_service_database` | ❌ | | 4 | 0.382377 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.377430 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.377241 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -2788,7 +2778,7 @@ | 1 | 0.619236 | `get_azure_app_config_settings` | ✅ **EXPECTED** | | 2 | 0.496884 | `lock_unlock_azure_app_config_settings` | ❌ | | 3 | 0.413994 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.320934 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.320917 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.318437 | `get_azure_key_vault_secret_values` | ❌ | --- @@ -2823,7 +2813,7 @@ | 2 | 0.419225 | `lock_unlock_azure_app_config_settings` | ❌ | | 3 | 0.386233 | `get_azure_app_config_settings` | ❌ | | 4 | 0.246854 | `edit_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.245595 | `add_azure_app_service_database` | ❌ | +| 5 | 0.245528 | `add_azure_app_service_database` | ❌ | --- @@ -2836,11 +2826,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.454235 | `lock_unlock_azure_app_config_settings` | ❌ | -| 2 | 0.420016 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.418392 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | -| 4 | 0.282997 | `update_azure_managed_lustre_filesystems` | ❌ | -| 5 | 0.270549 | `add_azure_app_service_database` | ❌ | +| 1 | 0.454141 | `lock_unlock_azure_app_config_settings` | ❌ | +| 2 | 0.419890 | `get_azure_app_config_settings` | ❌ | +| 3 | 0.418151 | `edit_azure_app_config_settings` | ✅ **EXPECTED** | +| 4 | 0.282688 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.270519 | `add_azure_app_service_database` | ❌ | --- @@ -2856,7 +2846,7 @@ | 1 | 0.523446 | `lock_unlock_azure_app_config_settings` | ✅ **EXPECTED** | | 2 | 0.367924 | `get_azure_app_config_settings` | ❌ | | 3 | 0.324653 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.243923 | `add_azure_app_service_database` | ❌ | +| 4 | 0.243851 | `add_azure_app_service_database` | ❌ | | 5 | 0.238376 | `update_azure_managed_lustre_filesystems` | ❌ | --- @@ -2874,7 +2864,7 @@ | 2 | 0.393938 | `get_azure_app_config_settings` | ❌ | | 3 | 0.339108 | `edit_azure_app_config_settings` | ❌ | | 4 | 0.276616 | `get_azure_key_vault_secret_values` | ❌ | -| 5 | 0.242957 | `add_azure_app_service_database` | ❌ | +| 5 | 0.242903 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3006,8 +2996,8 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546941 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | -| 2 | 0.541006 | `get_azure_capacity` | ❌ | +| 1 | 0.557893 | `get_azure_capacity` | ❌ | +| 2 | 0.546941 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | | 3 | 0.500223 | `browse_azure_marketplace_products` | ❌ | | 4 | 0.477388 | `get_azure_best_practices` | ❌ | | 5 | 0.474918 | `get_azure_resource_and_app_health_status` | ❌ | @@ -3026,7 +3016,7 @@ | 1 | 0.536483 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | | 2 | 0.511039 | `get_azure_best_practices` | ❌ | | 3 | 0.490293 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.476949 | `get_azure_capacity` | ❌ | +| 4 | 0.487381 | `get_azure_capacity` | ❌ | | 5 | 0.463219 | `deploy_azure_resources_and_applications` | ❌ | --- @@ -3042,8 +3032,8 @@ |------|-------|------|--------| | 1 | 0.592611 | `audit_azure_resources_compliance` | ✅ **EXPECTED** | | 2 | 0.508765 | `get_azure_best_practices` | ❌ | -| 3 | 0.492553 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.490633 | `get_azure_capacity` | ❌ | +| 3 | 0.502901 | `get_azure_capacity` | ❌ | +| 4 | 0.492553 | `browse_azure_marketplace_products` | ❌ | | 5 | 0.455630 | `deploy_azure_resources_and_applications` | ❌ | --- @@ -3057,11 +3047,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.559532 | `create_azure_storage` | ❌ | -| 2 | 0.490462 | `get_azure_storage_details` | ❌ | -| 3 | 0.486254 | `generate_azure_cli_commands` | ✅ **EXPECTED** | -| 4 | 0.428312 | `install_azure_cli_extensions` | ❌ | -| 5 | 0.417789 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.559527 | `create_azure_storage` | ❌ | +| 2 | 0.490440 | `get_azure_storage_details` | ❌ | +| 3 | 0.486197 | `generate_azure_cli_commands` | ✅ **EXPECTED** | +| 4 | 0.428212 | `install_azure_cli_extensions` | ❌ | +| 5 | 0.417777 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3075,7 +3065,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.505461 | `generate_azure_cli_commands` | ✅ **EXPECTED** | -| 2 | 0.466697 | `install_azure_cli_extensions` | ❌ | +| 2 | 0.466689 | `install_azure_cli_extensions` | ❌ | | 3 | 0.460094 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.439158 | `browse_azure_marketplace_products` | ❌ | | 5 | 0.433504 | `get_azure_virtual_desktop_details` | ❌ | @@ -3108,11 +3098,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.734114 | `get_azure_security_configurations` | ✅ **EXPECTED** | -| 2 | 0.460374 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.414713 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.356351 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.341765 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.734080 | `get_azure_security_configurations` | ✅ **EXPECTED** | +| 2 | 0.460399 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.414687 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.365074 | `get_azure_capacity` | ❌ | +| 5 | 0.356352 | `browse_azure_marketplace_products` | ❌ | --- @@ -3127,9 +3117,9 @@ |------|-------|------|--------| | 1 | 0.702749 | `get_azure_security_configurations` | ✅ **EXPECTED** | | 2 | 0.485211 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.431017 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.388410 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.357972 | `get_azure_capacity` | ❌ | +| 3 | 0.431073 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.400579 | `get_azure_capacity` | ❌ | +| 5 | 0.388410 | `browse_azure_marketplace_products` | ❌ | --- @@ -3142,11 +3132,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533833 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.508789 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.465713 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.426342 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.393933 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.533821 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.508482 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.465773 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.426168 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.393742 | `get_azure_app_config_settings` | ❌ | --- @@ -3159,11 +3149,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.459981 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.447387 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.393272 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.343733 | `create_azure_key_vault_secrets` | ❌ | -| 5 | 0.342376 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.459787 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.447147 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.393173 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.343627 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.342336 | `import_azure_key_vault_certificates` | ❌ | --- @@ -3227,11 +3217,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510037 | `get_azure_app_config_settings` | ❌ | -| 2 | 0.469247 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.446870 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.391544 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.377218 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.509964 | `get_azure_app_config_settings` | ❌ | +| 2 | 0.469179 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.446761 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.391479 | `lock_unlock_azure_app_config_settings` | ❌ | +| 5 | 0.377164 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3380,11 +3370,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.526748 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.521132 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.458697 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.449969 | `get_azure_key_vault_secret_values` | ❌ | -| 5 | 0.352703 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.526739 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.521127 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.458662 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.449940 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.352644 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3465,11 +3455,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543492 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.508166 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.454602 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.453808 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.386522 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.543362 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.508426 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.454583 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.453867 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.386384 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3482,11 +3472,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552937 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.535104 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.480107 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.475447 | `get_azure_key_vault_secret_values` | ❌ | -| 5 | 0.430616 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.552928 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.535077 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.480063 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.475424 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.430555 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3499,11 +3489,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527499 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.490655 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.460251 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.432964 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.409334 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.527455 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.490672 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.460248 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.432966 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.409324 | `get_azure_app_config_settings` | ❌ | --- @@ -3516,11 +3506,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.480327 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.471396 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.426126 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.395013 | `create_azure_key_vault_secrets` | ❌ | -| 5 | 0.377245 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.480223 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.471361 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.426201 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.394979 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.377132 | `import_azure_key_vault_certificates` | ❌ | --- @@ -3533,11 +3523,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.464726 | `get_azure_key_vault_secret_values` | ❌ | -| 2 | 0.449853 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.407815 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.399191 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.379916 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.464736 | `get_azure_key_vault_secret_values` | ❌ | +| 2 | 0.449713 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.407701 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.399319 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.379740 | `create_azure_key_vault_items` | ❌ | --- @@ -3584,11 +3574,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595761 | `create_azure_key_vault_secrets` | ❌ | -| 2 | 0.581393 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.517120 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 4 | 0.434361 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.427298 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.595751 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.581382 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.517138 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.434379 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.427291 | `import_azure_key_vault_certificates` | ❌ | --- @@ -3618,11 +3608,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.551081 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 2 | 0.543959 | `create_azure_key_vault_secrets` | ❌ | -| 3 | 0.534076 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.520839 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.484809 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.551036 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 2 | 0.543936 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.534063 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.520805 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.484768 | `import_azure_key_vault_certificates` | ❌ | --- @@ -3635,11 +3625,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622269 | `create_azure_key_vault_secrets` | ❌ | -| 2 | 0.581377 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.564774 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 4 | 0.495588 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.438784 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.622270 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.581477 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.564796 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 4 | 0.495618 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.438877 | `import_azure_key_vault_certificates` | ❌ | --- @@ -3652,11 +3642,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498669 | `get_azure_app_config_settings` | ❌ | -| 2 | 0.468556 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.448361 | `lock_unlock_azure_app_config_settings` | ❌ | -| 4 | 0.444399 | `get_azure_key_vault_items` | ✅ **EXPECTED** | -| 5 | 0.409423 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.498642 | `get_azure_app_config_settings` | ❌ | +| 2 | 0.468577 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.448342 | `lock_unlock_azure_app_config_settings` | ❌ | +| 4 | 0.444459 | `get_azure_key_vault_items` | ✅ **EXPECTED** | +| 5 | 0.409443 | `create_azure_key_vault_secrets` | ❌ | --- @@ -3669,11 +3659,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.586421 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | -| 2 | 0.526749 | `get_azure_key_vault_items` | ❌ | -| 3 | 0.517989 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.440736 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.389603 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.586366 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.526985 | `get_azure_key_vault_items` | ❌ | +| 3 | 0.518051 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.440667 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.389702 | `create_azure_key_vault_items` | ❌ | --- @@ -3720,11 +3710,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542793 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | -| 2 | 0.507689 | `get_azure_key_vault_items` | ❌ | -| 3 | 0.493634 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.432729 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.375435 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.542116 | `get_azure_key_vault_secret_values` | ✅ **EXPECTED** | +| 2 | 0.507830 | `get_azure_key_vault_items` | ❌ | +| 3 | 0.493688 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.433356 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.375736 | `create_azure_key_vault_items` | ❌ | --- @@ -3771,11 +3761,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527822 | `create_azure_key_vault_secrets` | ❌ | -| 2 | 0.506017 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.417486 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.354703 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.330867 | `get_azure_key_vault_secret_values` | ❌ | +| 1 | 0.527766 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.505954 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.417468 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.354704 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.330865 | `get_azure_key_vault_secret_values` | ❌ | --- @@ -3822,11 +3812,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.545276 | `create_azure_key_vault_secrets` | ❌ | -| 2 | 0.534559 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.437082 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.391019 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.372312 | `get_azure_key_vault_secret_values` | ❌ | +| 1 | 0.545338 | `create_azure_key_vault_secrets` | ❌ | +| 2 | 0.534535 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.437054 | `import_azure_key_vault_certificates` | ❌ | +| 4 | 0.390812 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.372207 | `get_azure_key_vault_secret_values` | ❌ | --- @@ -3890,10 +3880,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555686 | `import_azure_key_vault_certificates` | ❌ | -| 2 | 0.554286 | `create_azure_key_vault_items` | ✅ **EXPECTED** | -| 3 | 0.500237 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.410230 | `get_azure_key_vault_items` | ❌ | +| 1 | 0.555645 | `import_azure_key_vault_certificates` | ❌ | +| 2 | 0.554305 | `create_azure_key_vault_items` | ✅ **EXPECTED** | +| 3 | 0.500254 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.410218 | `get_azure_key_vault_items` | ❌ | | 5 | 0.325473 | `get_azure_key_vault_secret_values` | ❌ | --- @@ -3924,11 +3914,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.629763 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | -| 2 | 0.479387 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.407897 | `get_azure_key_vault_secret_values` | ❌ | -| 4 | 0.391544 | `import_azure_key_vault_certificates` | ❌ | -| 5 | 0.363090 | `append_azure_confidential_ledger_entries` | ❌ | +| 1 | 0.629675 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.479338 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.407829 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.391533 | `import_azure_key_vault_certificates` | ❌ | +| 5 | 0.363103 | `append_azure_confidential_ledger_entries` | ❌ | --- @@ -3958,11 +3948,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572358 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | -| 2 | 0.454589 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.415391 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.387428 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.357411 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.572357 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.454539 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.415396 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.387475 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.357474 | `lock_unlock_azure_app_config_settings` | ❌ | --- @@ -3975,10 +3965,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.588795 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | -| 2 | 0.475056 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.452602 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.402382 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.588807 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.475024 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.452599 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.402378 | `import_azure_key_vault_certificates` | ❌ | | 5 | 0.395541 | `get_azure_key_vault_items` | ❌ | --- @@ -3992,11 +3982,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.514668 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | -| 2 | 0.453316 | `get_azure_key_vault_secret_values` | ❌ | -| 3 | 0.390764 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.376214 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.365562 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.514684 | `create_azure_key_vault_secrets` | ✅ **EXPECTED** | +| 2 | 0.453288 | `get_azure_key_vault_secret_values` | ❌ | +| 3 | 0.390780 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.376210 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.365541 | `lock_unlock_azure_app_config_settings` | ❌ | --- @@ -4060,11 +4050,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630975 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | -| 2 | 0.464828 | `create_azure_key_vault_items` | ❌ | -| 3 | 0.419529 | `get_azure_key_vault_items` | ❌ | -| 4 | 0.390232 | `get_azure_key_vault_secret_values` | ❌ | -| 5 | 0.388920 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.630847 | `import_azure_key_vault_certificates` | ✅ **EXPECTED** | +| 2 | 0.464585 | `create_azure_key_vault_items` | ❌ | +| 3 | 0.419381 | `get_azure_key_vault_items` | ❌ | +| 4 | 0.390178 | `get_azure_key_vault_secret_values` | ❌ | +| 5 | 0.388744 | `create_azure_key_vault_secrets` | ❌ | --- @@ -4097,8 +4087,8 @@ | 1 | 0.735005 | `get_azure_best_practices` | ✅ **EXPECTED** | | 2 | 0.524465 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.439480 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.436925 | `get_azure_capacity` | ❌ | -| 5 | 0.436798 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.436798 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.435782 | `get_azure_app_config_settings` | ❌ | --- @@ -4114,7 +4104,7 @@ | 1 | 0.690117 | `get_azure_best_practices` | ✅ **EXPECTED** | | 2 | 0.539669 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.508718 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.483779 | `get_azure_capacity` | ❌ | +| 4 | 0.478557 | `get_azure_capacity` | ❌ | | 5 | 0.460726 | `get_azure_storage_details` | ❌ | --- @@ -4128,11 +4118,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.713406 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.529617 | `deploy_azure_resources_and_applications` | ❌ | -| 3 | 0.470109 | `design_azure_architecture` | ❌ | -| 4 | 0.437561 | `generate_azure_cli_commands` | ❌ | -| 5 | 0.435613 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.713392 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.529580 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.470093 | `design_azure_architecture` | ❌ | +| 4 | 0.437567 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.435577 | `browse_azure_marketplace_products` | ❌ | --- @@ -4147,9 +4137,9 @@ |------|-------|------|--------| | 1 | 0.683437 | `get_azure_best_practices` | ✅ **EXPECTED** | | 2 | 0.614180 | `deploy_azure_resources_and_applications` | ❌ | -| 3 | 0.475963 | `execute_azure_deployments` | ❌ | -| 4 | 0.465496 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.450131 | `get_azure_capacity` | ❌ | +| 3 | 0.465496 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.446368 | `get_azure_capacity` | ❌ | +| 5 | 0.427992 | `generate_azure_cli_commands` | ❌ | --- @@ -4166,7 +4156,7 @@ | 2 | 0.557930 | `get_azure_app_resource_details` | ❌ | | 3 | 0.505735 | `deploy_azure_resources_and_applications` | ❌ | | 4 | 0.443359 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.431202 | `get_azure_capacity` | ❌ | +| 5 | 0.432776 | `get_azure_capacity` | ❌ | --- @@ -4183,7 +4173,7 @@ | 2 | 0.499419 | `get_azure_app_resource_details` | ❌ | | 3 | 0.480287 | `deploy_azure_resources_and_applications` | ❌ | | 4 | 0.425068 | `generate_azure_cli_commands` | ❌ | -| 5 | 0.410038 | `execute_azure_deployments` | ❌ | +| 5 | 0.397939 | `design_azure_architecture` | ❌ | --- @@ -4196,11 +4186,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.675326 | `get_azure_best_practices` | ✅ **EXPECTED** | -| 2 | 0.570982 | `deploy_azure_resources_and_applications` | ❌ | -| 3 | 0.537910 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.460530 | `execute_azure_deployments` | ❌ | -| 5 | 0.415282 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.675358 | `get_azure_best_practices` | ✅ **EXPECTED** | +| 2 | 0.571007 | `deploy_azure_resources_and_applications` | ❌ | +| 3 | 0.537884 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.415218 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.413866 | `get_azure_capacity` | ❌ | --- @@ -4217,7 +4207,7 @@ | 2 | 0.518435 | `deploy_azure_resources_and_applications` | ❌ | | 3 | 0.424667 | `browse_azure_marketplace_products` | ❌ | | 4 | 0.418814 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.417729 | `execute_azure_deployments` | ❌ | +| 5 | 0.417600 | `get_azure_app_resource_details` | ❌ | --- @@ -4267,8 +4257,8 @@ | 1 | 0.628027 | `get_azure_best_practices` | ✅ **EXPECTED** | | 2 | 0.475164 | `get_azure_app_resource_details` | ❌ | | 3 | 0.453910 | `deploy_azure_resources_and_applications` | ❌ | -| 4 | 0.380736 | `execute_azure_deployments` | ❌ | -| 5 | 0.379701 | `get_azure_capacity` | ❌ | +| 4 | 0.378761 | `get_azure_capacity` | ❌ | +| 5 | 0.363076 | `generate_azure_cli_commands` | ❌ | --- @@ -4335,8 +4325,8 @@ | 1 | 0.342485 | `browse_azure_marketplace_products` | ❌ | | 2 | 0.289308 | `design_azure_architecture` | ✅ **EXPECTED** | | 3 | 0.276203 | `deploy_azure_resources_and_applications` | ❌ | -| 4 | 0.251655 | `add_azure_app_service_database` | ❌ | -| 5 | 0.236505 | `execute_azure_deployments` | ❌ | +| 4 | 0.251719 | `add_azure_app_service_database` | ❌ | +| 5 | 0.231297 | `get_azure_app_resource_details` | ❌ | --- @@ -4349,11 +4339,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.323447 | `upload_azure_storage_blobs` | ❌ | -| 2 | 0.305869 | `create_azure_storage` | ❌ | -| 3 | 0.296671 | `design_azure_architecture` | ✅ **EXPECTED** | -| 4 | 0.261376 | `get_azure_storage_details` | ❌ | -| 5 | 0.235389 | `update_azure_managed_lustre_filesystems` | ❌ | +| 1 | 0.323311 | `upload_azure_storage_blobs` | ❌ | +| 2 | 0.305813 | `create_azure_storage` | ❌ | +| 3 | 0.296166 | `design_azure_architecture` | ✅ **EXPECTED** | +| 4 | 0.261114 | `get_azure_storage_details` | ❌ | +| 5 | 0.235001 | `update_azure_managed_lustre_filesystems` | ❌ | --- @@ -4366,11 +4356,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609585 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 2 | 0.568056 | `create_azure_load_testing` | ❌ | +| 1 | 0.609260 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 2 | 0.567757 | `create_azure_load_testing` | ❌ | | 3 | 0.448056 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.366497 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.353481 | `create_azure_monitor_webtests` | ❌ | +| 4 | 0.366305 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 5 | 0.353288 | `create_azure_monitor_webtests` | ❌ | --- @@ -4400,11 +4390,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612755 | `create_azure_load_testing` | ❌ | -| 2 | 0.592683 | `get_azure_load_testing_details` | ✅ **EXPECTED** | -| 3 | 0.421766 | `update_azure_load_testing_configurations` | ❌ | -| 4 | 0.362418 | `create_azure_monitor_webtests` | ❌ | -| 5 | 0.349180 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 1 | 0.612800 | `create_azure_load_testing` | ❌ | +| 2 | 0.592725 | `get_azure_load_testing_details` | ✅ **EXPECTED** | +| 3 | 0.421873 | `update_azure_load_testing_configurations` | ❌ | +| 4 | 0.362425 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.349117 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- @@ -4420,8 +4410,8 @@ | 1 | 0.669717 | `get_azure_load_testing_details` | ✅ **EXPECTED** | | 2 | 0.609875 | `create_azure_load_testing` | ❌ | | 3 | 0.493520 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.426767 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.421963 | `get_azure_capacity` | ❌ | +| 4 | 0.427548 | `get_azure_capacity` | ❌ | +| 5 | 0.426809 | `get_azure_messaging_service_details` | ❌ | --- @@ -4438,7 +4428,7 @@ | 2 | 0.491654 | `create_azure_monitor_webtests` | ❌ | | 3 | 0.431906 | `get_azure_load_testing_details` | ❌ | | 4 | 0.425527 | `update_azure_load_testing_configurations` | ❌ | -| 5 | 0.409332 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.409288 | `update_azure_monitor_webtests` | ❌ | --- @@ -4472,7 +4462,7 @@ | 2 | 0.496772 | `update_azure_load_testing_configurations` | ❌ | | 3 | 0.460907 | `get_azure_load_testing_details` | ❌ | | 4 | 0.420018 | `create_azure_monitor_webtests` | ❌ | -| 5 | 0.334626 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.334546 | `update_azure_monitor_webtests` | ❌ | --- @@ -4485,11 +4475,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577283 | `update_azure_load_testing_configurations` | ✅ **EXPECTED** | -| 2 | 0.501245 | `create_azure_load_testing` | ❌ | -| 3 | 0.443796 | `get_azure_load_testing_details` | ❌ | -| 4 | 0.369079 | `update_azure_monitor_webtests` | ❌ | -| 5 | 0.360601 | `rename_azure_sql_databases` | ❌ | +| 1 | 0.577419 | `update_azure_load_testing_configurations` | ✅ **EXPECTED** | +| 2 | 0.501316 | `create_azure_load_testing` | ❌ | +| 3 | 0.443800 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.369297 | `update_azure_monitor_webtests` | ❌ | +| 5 | 0.360753 | `rename_azure_sql_databases` | ❌ | --- @@ -4504,9 +4494,9 @@ |------|-------|------|--------| | 1 | 0.599211 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.449921 | `get_azure_signalr_details` | ❌ | -| 3 | 0.425558 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.418851 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.418790 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.443802 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.429913 | `deploy_azure_ai_models` | ❌ | +| 5 | 0.425558 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- @@ -4519,11 +4509,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.659122 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 2 | 0.646614 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.486623 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.442045 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.441523 | `get_azure_signalr_details` | ❌ | +| 1 | 0.659138 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 2 | 0.646622 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.486694 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.442025 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.441556 | `get_azure_signalr_details` | ❌ | --- @@ -4538,7 +4528,7 @@ |------|-------|------|--------| | 1 | 0.600909 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.585328 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 3 | 0.440329 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.440398 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.437073 | `get_azure_signalr_details` | ❌ | | 5 | 0.416989 | `get_azure_app_resource_details` | ❌ | @@ -4570,11 +4560,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521566 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 1 | 0.521635 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 2 | 0.510202 | `evaluate_azure_ai_foundry_agents` | ❌ | | 3 | 0.502701 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 4 | 0.498204 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.362174 | `use_azure_openai_models` | ❌ | +| 4 | 0.498233 | `connect_azure_ai_foundry_agents` | ❌ | +| 5 | 0.406924 | `deploy_azure_ai_models` | ❌ | --- @@ -4587,11 +4577,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.562287 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.448496 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.440588 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.412586 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.394938 | `execute_azure_deployments` | ❌ | +| 1 | 0.579535 | `deploy_azure_ai_models` | ❌ | +| 2 | 0.562287 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.448518 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.440588 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.412607 | `connect_azure_ai_foundry_agents` | ❌ | --- @@ -4605,10 +4595,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.504981 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.403945 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.398748 | `use_azure_openai_models` | ❌ | -| 4 | 0.397053 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.367885 | `connect_azure_ai_foundry_agents` | ❌ | +| 2 | 0.482742 | `deploy_azure_ai_models` | ❌ | +| 3 | 0.403986 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.398748 | `use_azure_openai_models` | ❌ | +| 5 | 0.397053 | `evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -4622,10 +4612,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.579817 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.406256 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.405825 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.403903 | `connect_azure_ai_foundry_agents` | ❌ | -| 5 | 0.396690 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.418960 | `deploy_azure_ai_models` | ❌ | +| 3 | 0.406256 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.405825 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.403897 | `connect_azure_ai_foundry_agents` | ❌ | --- @@ -4641,8 +4631,8 @@ | 1 | 0.603264 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.498031 | `use_azure_openai_models` | ❌ | | 3 | 0.484432 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.456947 | `get_azure_capacity` | ❌ | -| 5 | 0.435210 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.443802 | `deploy_azure_ai_models` | ❌ | +| 5 | 0.435368 | `get_azure_capacity` | ❌ | --- @@ -4656,7 +4646,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.534733 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.453699 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.454264 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.444086 | `browse_azure_marketplace_products` | ❌ | | 4 | 0.419389 | `get_azure_app_resource_details` | ❌ | | 5 | 0.416112 | `get_azure_signalr_details` | ❌ | @@ -4675,7 +4665,7 @@ | 1 | 0.482325 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.398786 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 3 | 0.364422 | `get_azure_signalr_details` | ❌ | -| 4 | 0.357199 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.357577 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.356614 | `get_azure_security_configurations` | ❌ | --- @@ -4692,7 +4682,7 @@ | 1 | 0.667963 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.621099 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 3 | 0.450304 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.444213 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.444408 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.401759 | `get_azure_databases_details` | ❌ | --- @@ -4709,7 +4699,7 @@ | 1 | 0.531138 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.448783 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 3 | 0.337241 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.322260 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.323032 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.320822 | `get_azure_container_details` | ❌ | --- @@ -4726,8 +4716,8 @@ | 1 | 0.515763 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.410647 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 3 | 0.404503 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.396550 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.366440 | `use_azure_openai_models` | ❌ | +| 4 | 0.396554 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.395810 | `deploy_azure_ai_models` | ❌ | --- @@ -4742,7 +4732,7 @@ |------|-------|------|--------| | 1 | 0.654111 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.615349 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 3 | 0.446649 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.446627 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.422491 | `get_azure_data_explorer_kusto_details` | ❌ | | 5 | 0.416060 | `get_azure_signalr_details` | ❌ | @@ -4759,7 +4749,7 @@ |------|-------|------|--------| | 1 | 0.453376 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.433866 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 3 | 0.321039 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.321611 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.300443 | `get_azure_container_details` | ❌ | | 5 | 0.295001 | `get_azure_signalr_details` | ❌ | @@ -4777,7 +4767,7 @@ | 1 | 0.507589 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.425742 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 3 | 0.309441 | `get_azure_resource_and_app_health_status` | ❌ | -| 4 | 0.298713 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.298698 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 5 | 0.291702 | `use_azure_openai_models` | ❌ | --- @@ -4791,11 +4781,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552427 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.462315 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.458855 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.435445 | `use_azure_openai_models` | ❌ | -| 5 | 0.425410 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.599374 | `deploy_azure_ai_models` | ❌ | +| 2 | 0.552427 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.462343 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.458855 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.435445 | `use_azure_openai_models` | ❌ | --- @@ -4826,10 +4816,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.588933 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.472651 | `connect_azure_ai_foundry_agents` | ❌ | -| 3 | 0.467078 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.463356 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.424608 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 2 | 0.510164 | `deploy_azure_ai_models` | ❌ | +| 3 | 0.472600 | `connect_azure_ai_foundry_agents` | ❌ | +| 4 | 0.467078 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.463365 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -4842,11 +4832,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546302 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 2 | 0.534388 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.546352 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.534425 | `connect_azure_ai_foundry_agents` | ❌ | | 3 | 0.530860 | `evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.525204 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 5 | 0.421105 | `use_azure_openai_models` | ❌ | +| 5 | 0.429424 | `deploy_azure_ai_models` | ❌ | --- @@ -4859,11 +4849,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.522598 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.509261 | `use_azure_openai_models` | ❌ | -| 3 | 0.460083 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.459774 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.408332 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.555128 | `deploy_azure_ai_models` | ❌ | +| 2 | 0.522598 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 3 | 0.509261 | `use_azure_openai_models` | ❌ | +| 4 | 0.460129 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.459774 | `evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -4879,7 +4869,7 @@ | 1 | 0.543850 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.507116 | `browse_azure_marketplace_products` | ❌ | | 3 | 0.468386 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 4 | 0.447907 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.448711 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.425779 | `get_azure_app_resource_details` | ❌ | --- @@ -4893,11 +4883,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.494181 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 1 | 0.494174 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.450784 | `get_azure_signalr_details` | ❌ | -| 3 | 0.422378 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.417792 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.406409 | `get_azure_data_explorer_kusto_details` | ❌ | +| 3 | 0.423479 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.417724 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.406388 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -4913,7 +4903,7 @@ | 1 | 0.470270 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.407167 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 3 | 0.378470 | `get_azure_signalr_details` | ❌ | -| 4 | 0.372832 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.373514 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.360971 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -4929,8 +4919,8 @@ |------|-------|------|--------| | 1 | 0.496619 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.435586 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.309789 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.302510 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.310044 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.302527 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 5 | 0.290298 | `get_azure_container_details` | ❌ | --- @@ -4946,7 +4936,7 @@ |------|-------|------|--------| | 1 | 0.685811 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.620774 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.463222 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.463308 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.439438 | `get_azure_data_explorer_kusto_details` | ❌ | | 5 | 0.410728 | `get_azure_signalr_details` | ❌ | @@ -4963,7 +4953,7 @@ |------|-------|------|--------| | 1 | 0.557843 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.456176 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.376595 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.377134 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.337980 | `get_azure_container_details` | ❌ | | 5 | 0.327510 | `get_azure_data_explorer_kusto_details` | ❌ | @@ -4978,11 +4968,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502676 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.463915 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 3 | 0.445706 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.423761 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 5 | 0.406387 | `use_azure_openai_models` | ❌ | +| 1 | 0.502671 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.463948 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.445783 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.423817 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 5 | 0.406434 | `use_azure_openai_models` | ❌ | --- @@ -4997,7 +4987,7 @@ |------|-------|------|--------| | 1 | 0.417775 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.373515 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.281319 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.282160 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.261925 | `get_azure_container_details` | ❌ | | 5 | 0.256054 | `get_azure_signalr_details` | ❌ | @@ -5012,11 +5002,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.642192 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 2 | 0.627001 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 3 | 0.459537 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.420278 | `get_azure_signalr_details` | ❌ | -| 5 | 0.417829 | `get_azure_resource_and_app_health_status` | ❌ | +| 1 | 0.642314 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 2 | 0.626649 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 3 | 0.459611 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.420373 | `get_azure_signalr_details` | ❌ | +| 5 | 0.417992 | `get_azure_resource_and_app_health_status` | ❌ | --- @@ -5029,11 +5019,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.465907 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 1 | 0.465949 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 2 | 0.459306 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.362531 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.321876 | `get_azure_signalr_details` | ❌ | -| 5 | 0.307885 | `get_azure_container_details` | ❌ | +| 3 | 0.363115 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.321897 | `get_azure_signalr_details` | ❌ | +| 5 | 0.307899 | `get_azure_container_details` | ❌ | --- @@ -5046,11 +5036,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.442176 | `use_azure_openai_models` | ❌ | -| 2 | 0.414312 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | -| 3 | 0.364747 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.351248 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.350036 | `execute_azure_deployments` | ❌ | +| 1 | 0.454384 | `deploy_azure_ai_models` | ❌ | +| 2 | 0.442176 | `use_azure_openai_models` | ❌ | +| 3 | 0.414312 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | +| 4 | 0.364747 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.351290 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -5065,9 +5055,9 @@ |------|-------|------|--------| | 1 | 0.478034 | `get_azure_ai_resources_details` | ✅ **EXPECTED** | | 2 | 0.412935 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 3 | 0.324853 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.324788 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.323317 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.343793 | `deploy_azure_ai_models` | ❌ | +| 4 | 0.324853 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.324788 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -5082,9 +5072,9 @@ |------|-------|------|--------| | 1 | 0.566491 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | | 2 | 0.464178 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.312268 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.312344 | `get_azure_messaging_service_details` | ❌ | | 4 | 0.305023 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.304985 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.304988 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- @@ -5099,9 +5089,9 @@ |------|-------|------|--------| | 1 | 0.564315 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | | 2 | 0.477702 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.330673 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.330689 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.327179 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.321587 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.321705 | `get_azure_messaging_service_details` | ❌ | --- @@ -5116,8 +5106,8 @@ |------|-------|------|--------| | 1 | 0.516858 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | | 2 | 0.465631 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.353651 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.324671 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.353677 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.324901 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.306547 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -5131,11 +5121,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.669285 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | -| 2 | 0.561779 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.450678 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.412118 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.390813 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.669464 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.561928 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.450609 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.412094 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.390953 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -5150,7 +5140,7 @@ |------|-------|------|--------| | 1 | 0.524146 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | | 2 | 0.399606 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.322797 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.322840 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.267991 | `evaluate_azure_ai_foundry_agents` | ❌ | | 5 | 0.264021 | `get_azure_data_explorer_kusto_details` | ❌ | @@ -5167,9 +5157,9 @@ |------|-------|------|--------| | 1 | 0.691488 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | | 2 | 0.627605 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.462042 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.462028 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.421014 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.415222 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.414851 | `get_azure_messaging_service_details` | ❌ | --- @@ -5182,11 +5172,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530839 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | -| 2 | 0.439695 | `get_azure_ai_resources_details` | ❌ | -| 3 | 0.308871 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.285910 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.282944 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.530883 | `retrieve_azure_ai_knowledge_base_content` | ✅ **EXPECTED** | +| 2 | 0.439737 | `get_azure_ai_resources_details` | ❌ | +| 3 | 0.308946 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.286399 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.282951 | `get_azure_data_explorer_kusto_details` | ❌ | --- @@ -5200,10 +5190,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.389976 | `use_azure_openai_models` | ✅ **EXPECTED** | -| 2 | 0.254256 | `send_azure_communication_messages` | ❌ | +| 2 | 0.254263 | `send_azure_communication_messages` | ❌ | | 3 | 0.205840 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.184631 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.150665 | `connect_azure_ai_foundry_agents` | ❌ | +| 4 | 0.184587 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.150659 | `connect_azure_ai_foundry_agents` | ❌ | --- @@ -5216,11 +5206,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517427 | `generate_azure_cli_commands` | ❌ | -| 2 | 0.472325 | `use_azure_openai_models` | ✅ **EXPECTED** | -| 3 | 0.390310 | `execute_azure_deployments` | ❌ | -| 4 | 0.388807 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.377388 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.517506 | `generate_azure_cli_commands` | ❌ | +| 2 | 0.472396 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 3 | 0.388792 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.377460 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.367700 | `install_azure_cli_extensions` | ❌ | --- @@ -5233,11 +5223,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.682124 | `use_azure_openai_models` | ✅ **EXPECTED** | -| 2 | 0.414187 | `recognize_speech_from_audio` | ❌ | -| 3 | 0.403186 | `retrieve_azure_ai_knowledge_base_content` | ❌ | -| 4 | 0.401946 | `generate_azure_cli_commands` | ❌ | -| 5 | 0.385329 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.682093 | `use_azure_openai_models` | ✅ **EXPECTED** | +| 2 | 0.414225 | `recognize_speech_from_audio` | ❌ | +| 3 | 0.403170 | `retrieve_azure_ai_knowledge_base_content` | ❌ | +| 4 | 0.401921 | `generate_azure_cli_commands` | ❌ | +| 5 | 0.398779 | `deploy_azure_ai_models` | ❌ | --- @@ -5267,11 +5257,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.575213 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 2 | 0.514602 | `connect_azure_ai_foundry_agents` | ✅ **EXPECTED** | +| 1 | 0.575247 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 2 | 0.514647 | `connect_azure_ai_foundry_agents` | ✅ **EXPECTED** | | 3 | 0.510270 | `evaluate_azure_ai_foundry_agents` | ❌ | | 4 | 0.495795 | `get_azure_ai_resources_details` | ❌ | -| 5 | 0.385756 | `use_azure_openai_models` | ❌ | +| 5 | 0.392199 | `deploy_azure_ai_models` | ❌ | --- @@ -5285,10 +5275,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.596620 | `evaluate_azure_ai_foundry_agents` | ❌ | -| 2 | 0.590123 | `query_and_evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | -| 3 | 0.423635 | `connect_azure_ai_foundry_agents` | ❌ | +| 2 | 0.590134 | `query_and_evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | +| 3 | 0.423700 | `connect_azure_ai_foundry_agents` | ❌ | | 4 | 0.383588 | `get_azure_ai_resources_details` | ❌ | -| 5 | 0.319233 | `use_azure_openai_models` | ❌ | +| 5 | 0.365679 | `deploy_azure_ai_models` | ❌ | --- @@ -5301,11 +5291,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.426012 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 1 | 0.425995 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 2 | 0.364688 | `evaluate_azure_ai_foundry_agents` | ✅ **EXPECTED** | | 3 | 0.262513 | `get_azure_resource_and_app_health_status` | ❌ | | 4 | 0.249584 | `get_azure_ai_resources_details` | ❌ | -| 5 | 0.238878 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.239002 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -5320,9 +5310,9 @@ |------|-------|------|--------| | 1 | 0.623815 | `get_azure_storage_details` | ✅ **EXPECTED** | | 2 | 0.475469 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.430956 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.426398 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.422049 | `create_azure_storage` | ❌ | +| 3 | 0.433314 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.430956 | `get_azure_app_resource_details` | ❌ | +| 5 | 0.427589 | `get_azure_messaging_service_details` | ❌ | --- @@ -5369,11 +5359,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.497314 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.463770 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.458536 | `create_azure_storage` | ❌ | -| 4 | 0.338651 | `get_azure_container_details` | ❌ | -| 5 | 0.309725 | `get_azure_security_configurations` | ❌ | +| 1 | 0.497304 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.463738 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.458514 | `create_azure_storage` | ❌ | +| 4 | 0.338640 | `get_azure_container_details` | ❌ | +| 5 | 0.309742 | `get_azure_security_configurations` | ❌ | --- @@ -5387,10 +5377,10 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.611409 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.444366 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.441300 | `create_azure_storage` | ❌ | -| 4 | 0.428352 | `get_azure_capacity` | ❌ | -| 5 | 0.400577 | `browse_azure_marketplace_products` | ❌ | +| 2 | 0.455104 | `get_azure_capacity` | ❌ | +| 3 | 0.444366 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.441300 | `create_azure_storage` | ❌ | +| 5 | 0.401717 | `get_azure_messaging_service_details` | ❌ | --- @@ -5407,7 +5397,7 @@ | 2 | 0.577262 | `create_azure_storage` | ❌ | | 3 | 0.573528 | `get_azure_storage_details` | ✅ **EXPECTED** | | 4 | 0.432052 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 5 | 0.421279 | `get_azure_capacity` | ❌ | +| 5 | 0.417417 | `get_azure_messaging_service_details` | ❌ | --- @@ -5423,7 +5413,7 @@ | 1 | 0.605946 | `update_azure_managed_lustre_filesystems` | ❌ | | 2 | 0.596710 | `create_azure_storage` | ❌ | | 3 | 0.593813 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 4 | 0.435301 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.436583 | `get_azure_messaging_service_details` | ❌ | | 5 | 0.431655 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- @@ -5441,7 +5431,7 @@ | 2 | 0.541443 | `update_azure_managed_lustre_filesystems` | ❌ | | 3 | 0.531024 | `create_azure_storage` | ❌ | | 4 | 0.463214 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.435044 | `get_azure_capacity` | ❌ | +| 5 | 0.412477 | `get_azure_capacity` | ❌ | --- @@ -5456,9 +5446,9 @@ |------|-------|------|--------| | 1 | 0.493767 | `get_azure_storage_details` | ✅ **EXPECTED** | | 2 | 0.426243 | `create_azure_storage` | ❌ | -| 3 | 0.388186 | `get_azure_capacity` | ❌ | -| 4 | 0.384612 | `get_azure_security_configurations` | ❌ | -| 5 | 0.381052 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.384612 | `get_azure_security_configurations` | ❌ | +| 4 | 0.382781 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.377358 | `get_azure_capacity` | ❌ | --- @@ -5509,7 +5499,7 @@ | 2 | 0.449230 | `get_azure_app_config_settings` | ❌ | | 3 | 0.419483 | `create_azure_storage` | ❌ | | 4 | 0.419250 | `get_azure_container_details` | ❌ | -| 5 | 0.414316 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.417457 | `get_azure_cache_for_redis_details` | ❌ | --- @@ -5522,11 +5512,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546667 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.457238 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.439803 | `create_azure_storage` | ❌ | -| 4 | 0.405897 | `get_azure_container_details` | ❌ | -| 5 | 0.348646 | `get_azure_app_config_settings` | ❌ | +| 1 | 0.546641 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.457241 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.439747 | `create_azure_storage` | ❌ | +| 4 | 0.405931 | `get_azure_container_details` | ❌ | +| 5 | 0.348687 | `get_azure_app_config_settings` | ❌ | --- @@ -5543,7 +5533,7 @@ | 2 | 0.450881 | `create_azure_storage` | ❌ | | 3 | 0.434495 | `get_azure_container_details` | ❌ | | 4 | 0.398764 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.368711 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.373892 | `get_azure_capacity` | ❌ | --- @@ -5559,8 +5549,8 @@ | 1 | 0.512980 | `get_azure_storage_details` | ✅ **EXPECTED** | | 2 | 0.430206 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 3 | 0.425170 | `create_azure_storage` | ❌ | -| 4 | 0.397401 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.385193 | `get_azure_security_configurations` | ❌ | +| 4 | 0.401776 | `get_azure_capacity` | ❌ | +| 5 | 0.397521 | `get_azure_database_admin_configuration_details` | ❌ | --- @@ -5573,11 +5563,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.584136 | `get_azure_storage_details` | ✅ **EXPECTED** | -| 2 | 0.543122 | `create_azure_storage` | ❌ | -| 3 | 0.527934 | `update_azure_managed_lustre_filesystems` | ❌ | -| 4 | 0.421820 | `get_azure_capacity` | ❌ | -| 5 | 0.332218 | `generate_azure_cli_commands` | ❌ | +| 1 | 0.584062 | `get_azure_storage_details` | ✅ **EXPECTED** | +| 2 | 0.543203 | `create_azure_storage` | ❌ | +| 3 | 0.528023 | `update_azure_managed_lustre_filesystems` | ❌ | +| 4 | 0.382127 | `get_azure_capacity` | ❌ | +| 5 | 0.331802 | `generate_azure_cli_commands` | ❌ | --- @@ -5593,8 +5583,8 @@ | 1 | 0.574214 | `get_azure_storage_details` | ✅ **EXPECTED** | | 2 | 0.504246 | `update_azure_managed_lustre_filesystems` | ❌ | | 3 | 0.472221 | `create_azure_storage` | ❌ | -| 4 | 0.393120 | `get_azure_capacity` | ❌ | -| 5 | 0.316611 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.368401 | `get_azure_capacity` | ❌ | +| 5 | 0.316710 | `publish_azure_eventgrid_events` | ❌ | --- @@ -5661,7 +5651,7 @@ | 1 | 0.521592 | `create_azure_storage` | ✅ **EXPECTED** | | 2 | 0.483638 | `create_azure_sql_databases_and_servers` | ❌ | | 3 | 0.468735 | `get_azure_storage_details` | ❌ | -| 4 | 0.406117 | `get_azure_capacity` | ❌ | +| 4 | 0.369974 | `get_azure_capacity` | ❌ | | 5 | 0.356649 | `create_azure_load_testing` | ❌ | --- @@ -5679,7 +5669,7 @@ | 2 | 0.626919 | `create_azure_storage` | ✅ **EXPECTED** | | 3 | 0.533351 | `get_azure_storage_details` | ❌ | | 4 | 0.363841 | `create_azure_sql_databases_and_servers` | ❌ | -| 5 | 0.339901 | `get_azure_capacity` | ❌ | +| 5 | 0.318408 | `create_azure_load_testing` | ❌ | --- @@ -5692,11 +5682,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532843 | `create_azure_storage` | ✅ **EXPECTED** | -| 2 | 0.487514 | `upload_azure_storage_blobs` | ❌ | -| 3 | 0.440567 | `get_azure_storage_details` | ❌ | -| 4 | 0.345015 | `create_azure_key_vault_secrets` | ❌ | -| 5 | 0.326599 | `get_azure_container_details` | ❌ | +| 1 | 0.532844 | `create_azure_storage` | ✅ **EXPECTED** | +| 2 | 0.487471 | `upload_azure_storage_blobs` | ❌ | +| 3 | 0.440527 | `get_azure_storage_details` | ❌ | +| 4 | 0.345035 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.326507 | `get_azure_container_details` | ❌ | --- @@ -5726,11 +5716,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.664565 | `update_azure_managed_lustre_filesystems` | ✅ **EXPECTED** | -| 2 | 0.449482 | `create_azure_storage` | ❌ | -| 3 | 0.368632 | `get_azure_storage_details` | ❌ | -| 4 | 0.327263 | `edit_azure_databases` | ❌ | -| 5 | 0.323886 | `edit_azure_workbooks` | ❌ | +| 1 | 0.664019 | `update_azure_managed_lustre_filesystems` | ✅ **EXPECTED** | +| 2 | 0.448808 | `create_azure_storage` | ❌ | +| 3 | 0.368488 | `get_azure_storage_details` | ❌ | +| 4 | 0.327730 | `edit_azure_databases` | ❌ | +| 5 | 0.323192 | `edit_azure_workbooks` | ❌ | --- @@ -5753,176 +5743,6 @@ ## Test 312 -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all access policies in the Redis Cache - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.598205 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.335003 | `get_azure_security_configurations` | ❌ | -| 3 | 0.304910 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.290564 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.267608 | `get_azure_container_details` | ❌ | - ---- - -## Test 313 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all databases in the Redis Cluster - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.470698 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.389069 | `get_azure_databases_details` | ❌ | -| 3 | 0.387732 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.296636 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.280567 | `get_azure_container_details` | ❌ | - ---- - -## Test 314 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all Redis Caches in my subscription - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.580574 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.364804 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.340629 | `get_azure_databases_details` | ❌ | -| 4 | 0.321291 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.310751 | `browse_azure_marketplace_products` | ❌ | - ---- - -## Test 315 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** List all Redis Clusters in my subscription - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.567687 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.435563 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.414456 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.396583 | `get_azure_container_details` | ❌ | -| 5 | 0.364785 | `get_azure_messaging_service_details` | ❌ | - ---- - -## Test 316 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me my Redis Caches - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.520310 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.290240 | `get_azure_databases_details` | ❌ | -| 3 | 0.261870 | `get_azure_container_details` | ❌ | -| 4 | 0.252356 | `get_azure_security_configurations` | ❌ | -| 5 | 0.246602 | `browse_azure_marketplace_products` | ❌ | - ---- - -## Test 317 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me my Redis Clusters - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.498314 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.354127 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.342390 | `get_azure_container_details` | ❌ | -| 4 | 0.298268 | `get_azure_databases_details` | ❌ | -| 5 | 0.272676 | `get_azure_database_admin_configuration_details` | ❌ | - ---- - -## Test 318 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the access policies in the Redis Cache - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.600094 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.322512 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.316775 | `get_azure_security_configurations` | ❌ | -| 4 | 0.309470 | `get_azure_key_vault_items` | ❌ | -| 5 | 0.305487 | `get_azure_app_config_settings` | ❌ | - ---- - -## Test 319 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the databases in the Redis Cluster - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.456878 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.379510 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.372182 | `get_azure_databases_details` | ❌ | -| 4 | 0.280782 | `get_azure_container_details` | ❌ | -| 5 | 0.277615 | `rename_azure_sql_databases` | ❌ | - ---- - -## Test 320 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the Redis Caches in my subscription - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.553755 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.360815 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.335247 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.333570 | `get_azure_databases_details` | ❌ | -| 5 | 0.308733 | `get_azure_container_details` | ❌ | - ---- - -## Test 321 - -**Expected Tool:** `get_azure_cache_for_redis_details` -**Prompt:** Show me the Redis Clusters in my subscription - -### Results - -| Rank | Score | Tool | Status | -|------|-------|------|--------| -| 1 | 0.538723 | `get_azure_cache_for_redis_details` | ✅ **EXPECTED** | -| 2 | 0.424900 | `get_azure_data_explorer_kusto_details` | ❌ | -| 3 | 0.415817 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.400228 | `get_azure_container_details` | ❌ | -| 5 | 0.364169 | `get_azure_databases_details` | ❌ | - ---- - -## Test 322 - **Expected Tool:** `browse_azure_marketplace_products` **Prompt:** Get details about marketplace product @@ -5933,12 +5753,12 @@ | 1 | 0.424825 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | | 2 | 0.376519 | `get_azure_app_config_settings` | ❌ | | 3 | 0.359701 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.324404 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.310262 | `get_azure_storage_details` | ❌ | +| 4 | 0.324878 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.323149 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 323 +## Test 313 **Expected Tool:** `browse_azure_marketplace_products` **Prompt:** Search for Microsoft products in the marketplace @@ -5955,7 +5775,7 @@ --- -## Test 324 +## Test 314 **Expected Tool:** `browse_azure_marketplace_products` **Prompt:** Show me marketplace products from publisher @@ -5965,14 +5785,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.492651 | `browse_azure_marketplace_products` | ✅ **EXPECTED** | -| 2 | 0.225495 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.226351 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.218384 | `get_azure_ai_resources_details` | ❌ | | 4 | 0.215330 | `retrieve_azure_ai_knowledge_base_content` | ❌ | | 5 | 0.210581 | `get_azure_workbooks_details` | ❌ | --- -## Test 325 +## Test 315 **Expected Tool:** `get_azure_capacity` **Prompt:** Check usage information for in region @@ -5981,15 +5801,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.484871 | `get_azure_capacity` | ✅ **EXPECTED** | +| 1 | 0.530253 | `get_azure_capacity` | ✅ **EXPECTED** | | 2 | 0.397452 | `get_azure_storage_details` | ❌ | | 3 | 0.353830 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.350140 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.350026 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.353146 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.350349 | `get_azure_messaging_service_details` | ❌ | --- -## Test 326 +## Test 316 **Expected Tool:** `get_azure_capacity` **Prompt:** Show me the available regions for these resource types @@ -5998,15 +5818,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.398404 | `get_azure_capacity` | ✅ **EXPECTED** | +| 1 | 0.430316 | `get_azure_capacity` | ✅ **EXPECTED** | | 2 | 0.332100 | `get_azure_load_testing_details` | ❌ | | 3 | 0.326530 | `get_azure_storage_details` | ❌ | -| 4 | 0.322856 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.313365 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.322400 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.317967 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 327 +## Test 317 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Get the details of my consumer group in my event hub , namespace , and resource group @@ -6015,7 +5835,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549915 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.550433 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 2 | 0.537248 | `edit_azure_data_analytics_resources` | ❌ | | 3 | 0.417076 | `get_azure_signalr_details` | ❌ | | 4 | 0.413392 | `get_azure_app_config_settings` | ❌ | @@ -6023,7 +5843,7 @@ --- -## Test 328 +## Test 318 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Get the details of my event hub in my namespace and resource group @@ -6032,15 +5852,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.554775 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.510810 | `edit_azure_data_analytics_resources` | ❌ | -| 3 | 0.453455 | `get_azure_signalr_details` | ❌ | -| 4 | 0.423322 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.415404 | `get_azure_app_resource_details` | ❌ | +| 1 | 0.555243 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.510649 | `edit_azure_data_analytics_resources` | ❌ | +| 3 | 0.453511 | `get_azure_signalr_details` | ❌ | +| 4 | 0.423299 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.415266 | `get_azure_app_resource_details` | ❌ | --- -## Test 329 +## Test 319 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Get the details of my namespace in my resource group @@ -6049,15 +5869,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508903 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.468746 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.451446 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.432774 | `get_azure_signalr_details` | ❌ | -| 5 | 0.429765 | `get_azure_storage_details` | ❌ | +| 1 | 0.509618 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.468594 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.458941 | `get_azure_cache_for_redis_details` | ❌ | +| 4 | 0.450924 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.432453 | `get_azure_signalr_details` | ❌ | --- -## Test 330 +## Test 320 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all consumer groups in my event hub in namespace @@ -6067,14 +5887,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.552301 | `edit_azure_data_analytics_resources` | ❌ | -| 2 | 0.469857 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.470393 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 3 | 0.379261 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.360262 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.360237 | `publish_azure_eventgrid_events` | ❌ | | 5 | 0.338916 | `get_azure_signalr_details` | ❌ | --- -## Test 331 +## Test 321 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Grid subscriptions in subscription @@ -6083,15 +5903,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.445063 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.431684 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.414496 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.350456 | `get_azure_security_configurations` | ❌ | -| 5 | 0.335742 | `edit_azure_data_analytics_resources` | ❌ | +| 1 | 0.445663 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.431671 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.414508 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.350435 | `get_azure_security_configurations` | ❌ | +| 5 | 0.335767 | `edit_azure_data_analytics_resources` | ❌ | --- -## Test 332 +## Test 322 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Grid topics in my subscription @@ -6100,15 +5920,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.514030 | `publish_azure_eventgrid_events` | ❌ | -| 2 | 0.510686 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.514479 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.511417 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 3 | 0.433919 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.380839 | `browse_azure_marketplace_products` | ❌ | | 5 | 0.375303 | `get_azure_security_configurations` | ❌ | --- -## Test 333 +## Test 323 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Grid topics in resource group in subscription @@ -6118,14 +5938,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.517196 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.473028 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.440138 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.473204 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.440107 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.367530 | `edit_azure_data_analytics_resources` | ❌ | | 5 | 0.337400 | `get_azure_load_testing_details` | ❌ | --- -## Test 334 +## Test 324 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Grid topics in subscription @@ -6134,15 +5954,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.484028 | `publish_azure_eventgrid_events` | ❌ | -| 2 | 0.473797 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.484235 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.474480 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 3 | 0.397469 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.335248 | `edit_azure_data_analytics_resources` | ❌ | | 5 | 0.329802 | `get_azure_security_configurations` | ❌ | --- -## Test 335 +## Test 325 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Hubs in my namespace @@ -6152,14 +5972,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.538223 | `edit_azure_data_analytics_resources` | ❌ | -| 2 | 0.489438 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.381085 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.490081 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.381229 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.361397 | `get_azure_signalr_details` | ❌ | | 5 | 0.332270 | `get_azure_security_configurations` | ❌ | --- -## Test 336 +## Test 326 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List all Event Hubs namespaces in my subscription @@ -6168,15 +5988,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.517141 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.517926 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 2 | 0.508146 | `edit_azure_data_analytics_resources` | ❌ | | 3 | 0.415741 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.373001 | `get_azure_security_configurations` | ❌ | -| 5 | 0.366739 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.366988 | `publish_azure_eventgrid_events` | ❌ | --- -## Test 337 +## Test 327 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List Event Grid subscriptions for subscription in location @@ -6186,14 +6006,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.443596 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.439593 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.413135 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.440872 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.412665 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.352198 | `get_azure_storage_details` | ❌ | | 5 | 0.351500 | `get_azure_security_configurations` | ❌ | --- -## Test 338 +## Test 328 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List Event Grid subscriptions for topic in resource group @@ -6203,14 +6023,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.499123 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.494205 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.473537 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.494840 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.473120 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.366263 | `get_azure_security_configurations` | ❌ | | 5 | 0.347265 | `edit_azure_data_analytics_resources` | ❌ | --- -## Test 339 +## Test 329 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** List Event Grid subscriptions for topic in subscription @@ -6219,15 +6039,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.472417 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.469761 | `publish_azure_eventgrid_events` | ❌ | +| 1 | 0.473535 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.469527 | `publish_azure_eventgrid_events` | ❌ | | 3 | 0.422468 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.341614 | `get_azure_security_configurations` | ❌ | | 5 | 0.319377 | `browse_azure_marketplace_products` | ❌ | --- -## Test 340 +## Test 330 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show all Event Grid subscriptions in my subscription @@ -6236,15 +6056,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.480398 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 2 | 0.470532 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.453705 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.429371 | `get_azure_security_configurations` | ❌ | -| 5 | 0.402515 | `browse_azure_marketplace_products` | ❌ | +| 1 | 0.481543 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 2 | 0.470487 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.453733 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.429263 | `get_azure_security_configurations` | ❌ | +| 5 | 0.402453 | `browse_azure_marketplace_products` | ❌ | --- -## Test 341 +## Test 331 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show Event Grid subscriptions in resource group in subscription @@ -6254,14 +6074,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.566739 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 2 | 0.483011 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.406254 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.483454 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.405819 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.383949 | `get_azure_security_configurations` | ❌ | | 5 | 0.375086 | `edit_azure_data_analytics_resources` | ❌ | --- -## Test 342 +## Test 332 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show me all Event Grid subscriptions for topic @@ -6270,15 +6090,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498244 | `publish_azure_eventgrid_events` | ❌ | -| 2 | 0.475101 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.498127 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.476345 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 3 | 0.404707 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.344019 | `get_azure_security_configurations` | ❌ | | 5 | 0.337543 | `browse_azure_marketplace_products` | ❌ | --- -## Test 343 +## Test 333 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show me the details of service bus queue @@ -6287,15 +6107,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577545 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.577320 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 2 | 0.422716 | `get_azure_signalr_details` | ❌ | | 3 | 0.384758 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.365590 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.365499 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.364952 | `get_azure_app_config_settings` | ❌ | --- -## Test 344 +## Test 334 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show me the details of service bus subscription @@ -6304,7 +6124,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.565484 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.565533 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 2 | 0.432405 | `get_azure_signalr_details` | ❌ | | 3 | 0.397741 | `get_azure_app_resource_details` | ❌ | | 4 | 0.380696 | `get_azure_subscriptions_and_resource_groups` | ❌ | @@ -6312,7 +6132,7 @@ --- -## Test 345 +## Test 335 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show me the details of service bus topic @@ -6321,15 +6141,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.568951 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 1 | 0.569639 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | | 2 | 0.408956 | `get_azure_signalr_details` | ❌ | | 3 | 0.375835 | `get_azure_app_resource_details` | ❌ | | 4 | 0.363340 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.347442 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.347870 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 346 +## Test 336 **Expected Tool:** `get_azure_messaging_service_details` **Prompt:** Show me the Event Grid topics in my subscription @@ -6338,15 +6158,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.528995 | `publish_azure_eventgrid_events` | ❌ | -| 2 | 0.519422 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | -| 3 | 0.444005 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.417523 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.364752 | `get_azure_security_configurations` | ❌ | +| 1 | 0.529422 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.520326 | `get_azure_messaging_service_details` | ✅ **EXPECTED** | +| 3 | 0.444029 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.417590 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.364815 | `get_azure_security_configurations` | ❌ | --- -## Test 347 +## Test 337 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Create a new consumer group in my event hub , namespace , and resource group @@ -6356,14 +6176,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.567860 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.401880 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.380930 | `publish_azure_eventgrid_events` | ❌ | +| 2 | 0.401665 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.380821 | `publish_azure_eventgrid_events` | ❌ | | 4 | 0.343917 | `create_azure_key_vault_secrets` | ❌ | | 5 | 0.334668 | `create_azure_key_vault_items` | ❌ | --- -## Test 348 +## Test 338 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Create a new event hub in my namespace and resource group @@ -6372,15 +6192,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.537246 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.420610 | `publish_azure_eventgrid_events` | ❌ | -| 3 | 0.400466 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.365714 | `create_azure_key_vault_secrets` | ❌ | -| 5 | 0.364259 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.537219 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.420484 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.400204 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.365801 | `create_azure_key_vault_secrets` | ❌ | +| 5 | 0.364315 | `create_azure_key_vault_items` | ❌ | --- -## Test 349 +## Test 339 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Create an new namespace in my resource group @@ -6389,15 +6209,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.381243 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.380652 | `create_azure_key_vault_secrets` | ❌ | -| 3 | 0.352027 | `create_azure_key_vault_items` | ❌ | -| 4 | 0.329775 | `create_azure_load_testing` | ❌ | -| 5 | 0.324124 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.380823 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.380229 | `create_azure_key_vault_secrets` | ❌ | +| 3 | 0.351558 | `create_azure_key_vault_items` | ❌ | +| 4 | 0.329203 | `create_azure_load_testing` | ❌ | +| 5 | 0.323437 | `get_azure_messaging_service_details` | ❌ | --- -## Test 350 +## Test 340 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Delete my consumer group in my event hub , namespace , and resource group @@ -6407,14 +6227,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.669651 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.382770 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.330235 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.311527 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.305503 | `edit_azure_workbooks` | ❌ | +| 2 | 0.383213 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.330286 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.311518 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.305524 | `edit_azure_workbooks` | ❌ | --- -## Test 351 +## Test 341 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Delete my event hub in my namespace and resource group @@ -6423,15 +6243,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.658804 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.371664 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.359810 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.356070 | `edit_azure_app_config_settings` | ❌ | -| 5 | 0.340097 | `edit_azure_workbooks` | ❌ | +| 1 | 0.658817 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | +| 2 | 0.371998 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.359687 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.356153 | `edit_azure_app_config_settings` | ❌ | +| 5 | 0.340131 | `edit_azure_workbooks` | ❌ | --- -## Test 352 +## Test 342 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Delete my namespace in my resource group @@ -6448,7 +6268,7 @@ --- -## Test 353 +## Test 343 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Update my consumer group in my event hub , namespace , and resource group @@ -6458,14 +6278,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.738618 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | -| 2 | 0.439270 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.386803 | `update_azure_managed_lustre_filesystems` | ❌ | +| 2 | 0.439672 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.386800 | `update_azure_managed_lustre_filesystems` | ❌ | | 4 | 0.373932 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.362897 | `edit_azure_workbooks` | ❌ | +| 5 | 0.362896 | `edit_azure_workbooks` | ❌ | --- -## Test 354 +## Test 344 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Update my event hub in my namespace and resource group @@ -6476,13 +6296,13 @@ |------|-------|------|--------| | 1 | 0.708436 | `edit_azure_data_analytics_resources` | ✅ **EXPECTED** | | 2 | 0.440297 | `update_azure_managed_lustre_filesystems` | ❌ | -| 3 | 0.431338 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.423791 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.431754 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.423825 | `publish_azure_eventgrid_events` | ❌ | | 5 | 0.389301 | `edit_azure_workbooks` | ❌ | --- -## Test 355 +## Test 345 **Expected Tool:** `edit_azure_data_analytics_resources` **Prompt:** Update my namespace in my resource group @@ -6499,7 +6319,7 @@ --- -## Test 356 +## Test 346 **Expected Tool:** `publish_azure_eventgrid_events` **Prompt:** Publish an event to Event Grid topic using with the following data @@ -6508,15 +6328,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.732200 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | -| 2 | 0.364422 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.327560 | `edit_azure_data_analytics_resources` | ❌ | -| 4 | 0.289079 | `get_azure_best_practices` | ❌ | -| 5 | 0.286527 | `send_azure_communication_messages` | ❌ | +| 1 | 0.732024 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.365389 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.327559 | `edit_azure_data_analytics_resources` | ❌ | +| 4 | 0.289012 | `get_azure_best_practices` | ❌ | +| 5 | 0.286585 | `send_azure_communication_messages` | ❌ | --- -## Test 357 +## Test 347 **Expected Tool:** `publish_azure_eventgrid_events` **Prompt:** Publish event to my Event Grid topic with the following events @@ -6525,15 +6345,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.647380 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | -| 2 | 0.381287 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.647400 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.382138 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.352434 | `edit_azure_data_analytics_resources` | ❌ | -| 4 | 0.297256 | `send_azure_communication_messages` | ❌ | +| 4 | 0.297312 | `send_azure_communication_messages` | ❌ | | 5 | 0.294962 | `upload_azure_storage_blobs` | ❌ | --- -## Test 358 +## Test 348 **Expected Tool:** `publish_azure_eventgrid_events` **Prompt:** Send an event to Event Grid topic in resource group with @@ -6542,15 +6362,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.594047 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | -| 2 | 0.386591 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.593921 | `publish_azure_eventgrid_events` | ✅ **EXPECTED** | +| 2 | 0.386869 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.369923 | `edit_azure_data_analytics_resources` | ❌ | -| 4 | 0.295586 | `send_azure_communication_messages` | ❌ | +| 4 | 0.295613 | `send_azure_communication_messages` | ❌ | | 5 | 0.289053 | `get_azure_subscriptions_and_resource_groups` | ❌ | --- -## Test 359 +## Test 349 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** List all Data Explorer clusters in my subscription @@ -6562,12 +6382,12 @@ | 1 | 0.589762 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.413814 | `get_azure_databases_details` | ❌ | | 3 | 0.398499 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.385167 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.379203 | `get_azure_container_details` | ❌ | +| 4 | 0.379203 | `get_azure_container_details` | ❌ | +| 5 | 0.379108 | `browse_azure_marketplace_products` | ❌ | --- -## Test 360 +## Test 350 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** List all databases in the Data Explorer cluster @@ -6578,13 +6398,13 @@ |------|-------|------|--------| | 1 | 0.546030 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.462094 | `get_azure_databases_details` | ❌ | -| 3 | 0.337694 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.304208 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.295340 | `get_azure_container_details` | ❌ | +| 3 | 0.304208 | `rename_azure_sql_databases` | ❌ | +| 4 | 0.295340 | `get_azure_container_details` | ❌ | +| 5 | 0.281556 | `add_azure_app_service_database` | ❌ | --- -## Test 361 +## Test 351 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** List all tables in the Data Explorer database in cluster @@ -6595,13 +6415,13 @@ |------|-------|------|--------| | 1 | 0.526929 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.410927 | `get_azure_databases_details` | ❌ | -| 3 | 0.305623 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.256124 | `get_azure_container_details` | ❌ | -| 5 | 0.244911 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.256124 | `get_azure_container_details` | ❌ | +| 4 | 0.244911 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.243279 | `rename_azure_sql_databases` | ❌ | --- -## Test 362 +## Test 352 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me a data sample from the Data Explorer table in cluster @@ -6614,11 +6434,11 @@ | 2 | 0.313016 | `get_azure_databases_details` | ❌ | | 3 | 0.263223 | `get_azure_confidential_ledger_entries` | ❌ | | 4 | 0.248948 | `get_azure_container_details` | ❌ | -| 5 | 0.242279 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.229602 | `browse_azure_marketplace_products` | ❌ | --- -## Test 363 +## Test 353 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me all items that contain the word in the Data Explorer table in cluster @@ -6635,7 +6455,7 @@ --- -## Test 364 +## Test 354 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me my Data Explorer clusters @@ -6647,12 +6467,12 @@ | 1 | 0.533350 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.337173 | `get_azure_databases_details` | ❌ | | 3 | 0.337078 | `get_azure_container_details` | ❌ | -| 4 | 0.315555 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.305856 | `browse_azure_marketplace_products` | ❌ | +| 4 | 0.305856 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.288691 | `get_azure_resource_and_app_health_status` | ❌ | --- -## Test 365 +## Test 355 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me the Data Explorer clusters in my subscription @@ -6661,15 +6481,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.584974 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.420030 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.415732 | `get_azure_databases_details` | ❌ | -| 4 | 0.404725 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.400086 | `get_azure_container_details` | ❌ | +| 1 | 0.584941 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.420000 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.415745 | `get_azure_databases_details` | ❌ | +| 4 | 0.404683 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.400036 | `get_azure_container_details` | ❌ | --- -## Test 366 +## Test 356 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me the databases in the Data Explorer cluster @@ -6680,13 +6500,13 @@ |------|-------|------|--------| | 1 | 0.535152 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.443460 | `get_azure_databases_details` | ❌ | -| 3 | 0.327990 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.301627 | `get_azure_container_details` | ❌ | -| 5 | 0.295149 | `rename_azure_sql_databases` | ❌ | +| 3 | 0.301627 | `get_azure_container_details` | ❌ | +| 4 | 0.295149 | `rename_azure_sql_databases` | ❌ | +| 5 | 0.293066 | `add_azure_app_service_database` | ❌ | --- -## Test 367 +## Test 357 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me the details of the Data Explorer cluster @@ -6697,13 +6517,13 @@ |------|-------|------|--------| | 1 | 0.603734 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.416961 | `get_azure_container_details` | ❌ | -| 3 | 0.382586 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.365816 | `get_azure_databases_details` | ❌ | +| 3 | 0.365816 | `get_azure_databases_details` | ❌ | +| 4 | 0.363498 | `get_azure_cache_for_redis_details` | ❌ | | 5 | 0.363158 | `get_azure_signalr_details` | ❌ | --- -## Test 368 +## Test 358 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me the schema for table in the Data Explorer database in cluster @@ -6712,15 +6532,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.475347 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | -| 2 | 0.367386 | `get_azure_databases_details` | ❌ | -| 3 | 0.251481 | `get_azure_best_practices` | ❌ | -| 4 | 0.242454 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.242382 | `design_azure_architecture` | ❌ | +| 1 | 0.475232 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | +| 2 | 0.367299 | `get_azure_databases_details` | ❌ | +| 3 | 0.251410 | `get_azure_best_practices` | ❌ | +| 4 | 0.242322 | `design_azure_architecture` | ❌ | +| 5 | 0.241889 | `publish_azure_eventgrid_events` | ❌ | --- -## Test 369 +## Test 359 **Expected Tool:** `get_azure_data_explorer_kusto_details` **Prompt:** Show me the tables in the Data Explorer database in cluster @@ -6731,13 +6551,13 @@ |------|-------|------|--------| | 1 | 0.521279 | `get_azure_data_explorer_kusto_details` | ✅ **EXPECTED** | | 2 | 0.401926 | `get_azure_databases_details` | ❌ | -| 3 | 0.301626 | `get_azure_cache_for_redis_details` | ❌ | -| 4 | 0.271168 | `get_azure_container_details` | ❌ | -| 5 | 0.258457 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.271168 | `get_azure_container_details` | ❌ | +| 4 | 0.258235 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.253747 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 370 +## Test 360 **Expected Tool:** `create_azure_database_admin_configurations` **Prompt:** Add a firewall rule to allow access from IP range to for SQL server @@ -6749,12 +6569,12 @@ | 1 | 0.619667 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | | 2 | 0.497535 | `delete_azure_database_admin_configurations` | ❌ | | 3 | 0.339582 | `edit_azure_databases` | ❌ | -| 4 | 0.339238 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.339334 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.310935 | `rename_azure_sql_databases` | ❌ | --- -## Test 371 +## Test 361 **Expected Tool:** `create_azure_database_admin_configurations` **Prompt:** Create a firewall rule for my Azure SQL server @@ -6763,15 +6583,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.769949 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | -| 2 | 0.659650 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.476833 | `edit_azure_databases` | ❌ | -| 4 | 0.455132 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.452335 | `create_azure_sql_databases_and_servers` | ❌ | +| 1 | 0.769894 | `create_azure_database_admin_configurations` | ✅ **EXPECTED** | +| 2 | 0.659595 | `delete_azure_database_admin_configurations` | ❌ | +| 3 | 0.476817 | `edit_azure_databases` | ❌ | +| 4 | 0.455047 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.452299 | `create_azure_sql_databases_and_servers` | ❌ | --- -## Test 372 +## Test 362 **Expected Tool:** `create_azure_database_admin_configurations` **Prompt:** Create a new firewall rule named for SQL server @@ -6784,11 +6604,11 @@ | 2 | 0.546667 | `delete_azure_database_admin_configurations` | ❌ | | 3 | 0.409535 | `create_azure_sql_databases_and_servers` | ❌ | | 4 | 0.399140 | `rename_azure_sql_databases` | ❌ | -| 5 | 0.370061 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.370078 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 373 +## Test 363 **Expected Tool:** `delete_azure_database_admin_configurations` **Prompt:** Delete a firewall rule from my Azure SQL server @@ -6800,12 +6620,12 @@ | 1 | 0.725925 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | | 2 | 0.684225 | `create_azure_database_admin_configurations` | ❌ | | 3 | 0.507485 | `edit_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.446832 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.446796 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.434498 | `rename_azure_sql_databases` | ❌ | --- -## Test 374 +## Test 364 **Expected Tool:** `delete_azure_database_admin_configurations` **Prompt:** Delete firewall rule for SQL server @@ -6817,12 +6637,12 @@ | 1 | 0.691123 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | | 2 | 0.657272 | `create_azure_database_admin_configurations` | ❌ | | 3 | 0.411100 | `edit_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.410580 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.410644 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.398776 | `rename_azure_sql_databases` | ❌ | --- -## Test 375 +## Test 365 **Expected Tool:** `delete_azure_database_admin_configurations` **Prompt:** Remove the firewall rule from SQL server @@ -6834,12 +6654,12 @@ | 1 | 0.662278 | `delete_azure_database_admin_configurations` | ✅ **EXPECTED** | | 2 | 0.610044 | `create_azure_database_admin_configurations` | ❌ | | 3 | 0.374118 | `edit_azure_sql_databases_and_servers` | ❌ | -| 4 | 0.368243 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.368223 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.351139 | `rename_azure_sql_databases` | ❌ | --- -## Test 376 +## Test 366 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** List all elastic pools in SQL server @@ -6848,7 +6668,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550794 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 1 | 0.550801 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 2 | 0.437477 | `get_azure_databases_details` | ❌ | | 3 | 0.418188 | `create_azure_sql_databases_and_servers` | ❌ | | 4 | 0.411871 | `rename_azure_sql_databases` | ❌ | @@ -6856,7 +6676,7 @@ --- -## Test 377 +## Test 367 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** List all firewall rules for SQL server @@ -6867,13 +6687,13 @@ |------|-------|------|--------| | 1 | 0.659544 | `create_azure_database_admin_configurations` | ❌ | | 2 | 0.635949 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.509163 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 3 | 0.509175 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 4 | 0.344890 | `get_azure_security_configurations` | ❌ | | 5 | 0.332632 | `get_azure_databases_details` | ❌ | --- -## Test 378 +## Test 368 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** List Microsoft Entra ID administrators for SQL server @@ -6882,7 +6702,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498356 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 1 | 0.498412 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 2 | 0.362041 | `create_azure_database_admin_configurations` | ❌ | | 3 | 0.358939 | `get_azure_security_configurations` | ❌ | | 4 | 0.343594 | `get_azure_databases_details` | ❌ | @@ -6890,7 +6710,7 @@ --- -## Test 379 +## Test 369 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** Show me the elastic pools configured for SQL server @@ -6899,7 +6719,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.602459 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 1 | 0.602455 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 2 | 0.445269 | `create_azure_sql_databases_and_servers` | ❌ | | 3 | 0.424559 | `get_azure_databases_details` | ❌ | | 4 | 0.414641 | `edit_azure_databases` | ❌ | @@ -6907,7 +6727,7 @@ --- -## Test 380 +## Test 370 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** Show me the Entra ID administrators configured for SQL server @@ -6916,15 +6736,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498316 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 1 | 0.498397 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 2 | 0.325040 | `create_azure_database_admin_configurations` | ❌ | -| 3 | 0.300436 | `add_azure_app_service_database` | ❌ | +| 3 | 0.300449 | `add_azure_app_service_database` | ❌ | | 4 | 0.299005 | `get_azure_databases_details` | ❌ | | 5 | 0.294052 | `get_azure_security_configurations` | ❌ | --- -## Test 381 +## Test 371 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** Show me the firewall rules for SQL server @@ -6935,13 +6755,13 @@ |------|-------|------|--------| | 1 | 0.659102 | `create_azure_database_admin_configurations` | ❌ | | 2 | 0.611917 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.486395 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 3 | 0.486410 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 4 | 0.361115 | `edit_azure_databases` | ❌ | | 5 | 0.322908 | `get_azure_security_configurations` | ❌ | --- -## Test 382 +## Test 372 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** What elastic pools are available in my SQL server ? @@ -6950,15 +6770,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.515299 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.442723 | `create_azure_sql_databases_and_servers` | ❌ | -| 3 | 0.412957 | `edit_azure_databases` | ❌ | -| 4 | 0.411657 | `get_azure_databases_details` | ❌ | -| 5 | 0.380417 | `get_azure_capacity` | ❌ | +| 1 | 0.515387 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.442752 | `create_azure_sql_databases_and_servers` | ❌ | +| 3 | 0.412941 | `edit_azure_databases` | ❌ | +| 4 | 0.411655 | `get_azure_databases_details` | ❌ | +| 5 | 0.370955 | `get_azure_capacity` | ❌ | --- -## Test 383 +## Test 373 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** What firewall rules are configured for my SQL server ? @@ -6969,13 +6789,13 @@ |------|-------|------|--------| | 1 | 0.657075 | `create_azure_database_admin_configurations` | ❌ | | 2 | 0.595199 | `delete_azure_database_admin_configurations` | ❌ | -| 3 | 0.493189 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 3 | 0.493185 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | | 4 | 0.358803 | `edit_azure_databases` | ❌ | | 5 | 0.329180 | `edit_azure_sql_databases_and_servers` | ❌ | --- -## Test 384 +## Test 374 **Expected Tool:** `get_azure_database_admin_configuration_details` **Prompt:** What Microsoft Entra ID administrators are set up for my SQL server ? @@ -6984,15 +6804,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.451827 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | -| 2 | 0.340144 | `add_azure_app_service_database` | ❌ | +| 1 | 0.451771 | `get_azure_database_admin_configuration_details` | ✅ **EXPECTED** | +| 2 | 0.340143 | `add_azure_app_service_database` | ❌ | | 3 | 0.332608 | `create_azure_database_admin_configurations` | ❌ | | 4 | 0.331531 | `edit_azure_sql_databases_and_servers` | ❌ | | 5 | 0.331515 | `create_azure_sql_databases_and_servers` | ❌ | --- -## Test 385 +## Test 375 **Expected Tool:** `get_azure_container_details` **Prompt:** Get details for nodepool in AKS cluster in @@ -7001,15 +6821,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.591315 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.489551 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.461100 | `get_azure_database_admin_configuration_details` | ❌ | -| 4 | 0.438929 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.434421 | `get_azure_signalr_details` | ❌ | +| 1 | 0.591451 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.489539 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.461238 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.439012 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.434254 | `get_azure_signalr_details` | ❌ | --- -## Test 386 +## Test 376 **Expected Tool:** `get_azure_container_details` **Prompt:** Get the configuration of AKS cluster @@ -7020,13 +6840,13 @@ |------|-------|------|--------| | 1 | 0.525642 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.515594 | `get_azure_app_config_settings` | ❌ | -| 3 | 0.423935 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.423830 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.384930 | `get_azure_data_explorer_kusto_details` | ❌ | -| 5 | 0.379742 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.373640 | `lock_unlock_azure_app_config_settings` | ❌ | --- -## Test 387 +## Test 377 **Expected Tool:** `get_azure_container_details` **Prompt:** List all AKS clusters in my subscription @@ -7043,7 +6863,7 @@ --- -## Test 388 +## Test 378 **Expected Tool:** `get_azure_container_details` **Prompt:** List all Azure Container Registries in my subscription @@ -7060,7 +6880,7 @@ --- -## Test 389 +## Test 379 **Expected Tool:** `get_azure_container_details` **Prompt:** List all container registry repositories in my subscription @@ -7073,11 +6893,11 @@ | 2 | 0.394679 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 3 | 0.385192 | `get_azure_storage_details` | ❌ | | 4 | 0.351749 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.349969 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.345305 | `get_azure_security_configurations` | ❌ | --- -## Test 390 +## Test 380 **Expected Tool:** `get_azure_container_details` **Prompt:** List container registries in resource group @@ -7089,12 +6909,12 @@ | 1 | 0.489483 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.382508 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 3 | 0.365207 | `get_azure_storage_details` | ❌ | -| 4 | 0.356837 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.349921 | `get_azure_load_testing_details` | ❌ | +| 4 | 0.349921 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.338102 | `get_azure_security_configurations` | ❌ | --- -## Test 391 +## Test 381 **Expected Tool:** `get_azure_container_details` **Prompt:** List nodepools for AKS cluster in @@ -7103,15 +6923,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.542296 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.417414 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.385553 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.372743 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.371410 | `get_azure_data_explorer_kusto_details` | ❌ | +| 1 | 0.542243 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.417443 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.385526 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.371461 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.362639 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 392 +## Test 382 **Expected Tool:** `get_azure_container_details` **Prompt:** List repositories in the container registry @@ -7120,15 +6940,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.452303 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.285101 | `get_azure_cache_for_redis_details` | ❌ | -| 3 | 0.277147 | `get_azure_storage_details` | ❌ | -| 4 | 0.265399 | `get_azure_security_configurations` | ❌ | -| 5 | 0.261745 | `create_azure_storage` | ❌ | +| 1 | 0.452265 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.277163 | `get_azure_storage_details` | ❌ | +| 3 | 0.265440 | `get_azure_security_configurations` | ❌ | +| 4 | 0.261770 | `create_azure_storage` | ❌ | +| 5 | 0.260828 | `browse_azure_marketplace_products` | ❌ | --- -## Test 393 +## Test 383 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me my Azure Container Registries @@ -7141,11 +6961,11 @@ | 2 | 0.419226 | `browse_azure_marketplace_products` | ❌ | | 3 | 0.395307 | `get_azure_storage_details` | ❌ | | 4 | 0.390515 | `get_azure_security_configurations` | ❌ | -| 5 | 0.384917 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.378209 | `get_azure_databases_details` | ❌ | --- -## Test 394 +## Test 384 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me my Azure Kubernetes Service clusters @@ -7162,7 +6982,7 @@ --- -## Test 395 +## Test 385 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me my container registry repositories @@ -7175,11 +6995,11 @@ | 2 | 0.310577 | `get_azure_security_configurations` | ❌ | | 3 | 0.310265 | `get_azure_storage_details` | ❌ | | 4 | 0.294726 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.293685 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.292917 | `create_azure_storage` | ❌ | --- -## Test 396 +## Test 386 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the configuration for nodepool in AKS cluster in resource group @@ -7188,15 +7008,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516160 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.442585 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.410309 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.394415 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.360302 | `get_azure_signalr_details` | ❌ | +| 1 | 0.516023 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.442551 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.410279 | `get_azure_app_config_settings` | ❌ | +| 4 | 0.394278 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.360171 | `get_azure_signalr_details` | ❌ | --- -## Test 397 +## Test 387 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the container registries in my subscription @@ -7205,15 +7025,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.547872 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.432956 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.391672 | `get_azure_storage_details` | ❌ | -| 4 | 0.385551 | `browse_azure_marketplace_products` | ❌ | -| 5 | 0.361321 | `get_azure_messaging_service_details` | ❌ | +| 1 | 0.547796 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.433036 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 3 | 0.391606 | `get_azure_storage_details` | ❌ | +| 4 | 0.385491 | `browse_azure_marketplace_products` | ❌ | +| 5 | 0.362622 | `get_azure_messaging_service_details` | ❌ | --- -## Test 398 +## Test 388 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the container registries in resource group @@ -7224,13 +7044,13 @@ |------|-------|------|--------| | 1 | 0.510429 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.431510 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 3 | 0.355156 | `get_azure_cache_for_redis_details` | ❌ | +| 3 | 0.357940 | `get_azure_cache_for_redis_details` | ❌ | | 4 | 0.353883 | `get_azure_storage_details` | ❌ | -| 5 | 0.343124 | `get_azure_load_testing_details` | ❌ | +| 5 | 0.346827 | `get_azure_capacity` | ❌ | --- -## Test 399 +## Test 389 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the details of AKS cluster in resource group @@ -7242,12 +7062,12 @@ | 1 | 0.579908 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.459131 | `get_azure_signalr_details` | ❌ | | 3 | 0.444584 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.424458 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.424320 | `get_azure_virtual_desktop_details` | ❌ | +| 4 | 0.424320 | `get_azure_virtual_desktop_details` | ❌ | +| 5 | 0.422049 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 400 +## Test 390 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the network configuration for AKS cluster @@ -7259,12 +7079,12 @@ | 1 | 0.452245 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.375014 | `get_azure_app_config_settings` | ❌ | | 3 | 0.357934 | `get_azure_signalr_details` | ❌ | -| 4 | 0.343230 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.343177 | `get_azure_database_admin_configuration_details` | ❌ | | 5 | 0.322237 | `get_azure_data_explorer_kusto_details` | ❌ | --- -## Test 401 +## Test 391 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the nodepool list for AKS cluster in @@ -7276,12 +7096,12 @@ | 1 | 0.542863 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.412997 | `get_azure_virtual_desktop_details` | ❌ | | 3 | 0.387109 | `get_azure_data_explorer_kusto_details` | ❌ | -| 4 | 0.378755 | `get_azure_cache_for_redis_details` | ❌ | -| 5 | 0.373530 | `get_azure_database_admin_configuration_details` | ❌ | +| 4 | 0.373553 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.363902 | `get_azure_security_configurations` | ❌ | --- -## Test 402 +## Test 392 **Expected Tool:** `get_azure_container_details` **Prompt:** Show me the repositories in the container registry @@ -7291,14 +7111,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.463224 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.287737 | `get_azure_cache_for_redis_details` | ❌ | -| 3 | 0.277348 | `browse_azure_marketplace_products` | ❌ | -| 4 | 0.273850 | `get_azure_storage_details` | ❌ | -| 5 | 0.267446 | `get_azure_key_vault_items` | ❌ | +| 2 | 0.277348 | `browse_azure_marketplace_products` | ❌ | +| 3 | 0.273850 | `get_azure_storage_details` | ❌ | +| 4 | 0.267446 | `get_azure_key_vault_items` | ❌ | +| 5 | 0.258361 | `get_azure_databases_details` | ❌ | --- -## Test 403 +## Test 393 **Expected Tool:** `get_azure_container_details` **Prompt:** What AKS clusters do I have? @@ -7315,7 +7135,7 @@ --- -## Test 404 +## Test 394 **Expected Tool:** `get_azure_container_details` **Prompt:** What are the details of my AKS cluster in ? @@ -7332,7 +7152,7 @@ --- -## Test 405 +## Test 395 **Expected Tool:** `get_azure_container_details` **Prompt:** What is the setup of nodepool for AKS cluster in ? @@ -7343,13 +7163,13 @@ |------|-------|------|--------| | 1 | 0.489130 | `get_azure_container_details` | ✅ **EXPECTED** | | 2 | 0.347326 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.338797 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.338923 | `get_azure_database_admin_configuration_details` | ❌ | | 4 | 0.317925 | `get_azure_signalr_details` | ❌ | | 5 | 0.315321 | `generate_azure_cli_commands` | ❌ | --- -## Test 406 +## Test 396 **Expected Tool:** `get_azure_container_details` **Prompt:** What nodepools do I have for AKS cluster in @@ -7358,15 +7178,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532501 | `get_azure_container_details` | ✅ **EXPECTED** | -| 2 | 0.389162 | `get_azure_virtual_desktop_details` | ❌ | -| 3 | 0.362346 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.358739 | `get_azure_database_admin_configuration_details` | ❌ | -| 5 | 0.355086 | `get_azure_signalr_details` | ❌ | +| 1 | 0.532532 | `get_azure_container_details` | ✅ **EXPECTED** | +| 2 | 0.389232 | `get_azure_virtual_desktop_details` | ❌ | +| 3 | 0.362397 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.358841 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.355124 | `get_azure_signalr_details` | ❌ | --- -## Test 407 +## Test 397 **Expected Tool:** `get_azure_virtual_desktop_details` **Prompt:** List all host pools in my subscription @@ -7375,15 +7195,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550232 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.453703 | `get_azure_database_admin_configuration_details` | ❌ | -| 3 | 0.442934 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.412970 | `get_azure_container_details` | ❌ | -| 5 | 0.396066 | `get_azure_capacity` | ❌ | +| 1 | 0.550251 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | +| 2 | 0.453600 | `get_azure_database_admin_configuration_details` | ❌ | +| 3 | 0.442910 | `get_azure_subscriptions_and_resource_groups` | ❌ | +| 4 | 0.412967 | `get_azure_container_details` | ❌ | +| 5 | 0.394678 | `get_azure_messaging_service_details` | ❌ | --- -## Test 408 +## Test 398 **Expected Tool:** `get_azure_virtual_desktop_details` **Prompt:** List all session hosts in host pool @@ -7393,14 +7213,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.607532 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.364628 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.364740 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.319120 | `get_azure_security_configurations` | ❌ | | 4 | 0.312479 | `get_azure_signalr_details` | ❌ | | 5 | 0.307420 | `get_azure_container_details` | ❌ | --- -## Test 409 +## Test 399 **Expected Tool:** `get_azure_virtual_desktop_details` **Prompt:** List all user sessions on session host in host pool @@ -7410,14 +7230,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.611133 | `get_azure_virtual_desktop_details` | ✅ **EXPECTED** | -| 2 | 0.335733 | `get_azure_database_admin_configuration_details` | ❌ | +| 2 | 0.335895 | `get_azure_database_admin_configuration_details` | ❌ | | 3 | 0.313084 | `get_azure_security_configurations` | ❌ | | 4 | 0.283932 | `get_azure_signalr_details` | ❌ | | 5 | 0.265858 | `get_azure_container_details` | ❌ | --- -## Test 410 +## Test 400 **Expected Tool:** `get_azure_signalr_details` **Prompt:** Describe the SignalR runtime in resource group @@ -7427,14 +7247,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.711335 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.438870 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.438790 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.387114 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.370390 | `edit_azure_data_analytics_resources` | ❌ | -| 5 | 0.363831 | `get_azure_capacity` | ❌ | +| 4 | 0.373297 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.370390 | `edit_azure_data_analytics_resources` | ❌ | --- -## Test 411 +## Test 401 **Expected Tool:** `get_azure_signalr_details` **Prompt:** Get information about my SignalR runtime in @@ -7444,14 +7264,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.729127 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.467032 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.467563 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.452790 | `get_azure_app_resource_details` | ❌ | | 4 | 0.447727 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 5 | 0.433126 | `get_azure_app_config_settings` | ❌ | --- -## Test 412 +## Test 402 **Expected Tool:** `get_azure_signalr_details` **Prompt:** List all SignalRs in my subscription @@ -7461,14 +7281,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.497376 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.390874 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.391290 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.360482 | `get_azure_subscriptions_and_resource_groups` | ❌ | | 4 | 0.336211 | `get_azure_security_configurations` | ❌ | | 5 | 0.314121 | `browse_azure_marketplace_products` | ❌ | --- -## Test 413 +## Test 403 **Expected Tool:** `get_azure_signalr_details` **Prompt:** Show all the SignalRs information in @@ -7478,14 +7298,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.641102 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.499456 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.500333 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.461902 | `get_azure_subscriptions_and_resource_groups` | ❌ | -| 4 | 0.420390 | `get_azure_virtual_desktop_details` | ❌ | -| 5 | 0.410032 | `get_azure_capacity` | ❌ | +| 4 | 0.433484 | `get_azure_cache_for_redis_details` | ❌ | +| 5 | 0.420390 | `get_azure_virtual_desktop_details` | ❌ | --- -## Test 414 +## Test 404 **Expected Tool:** `get_azure_signalr_details` **Prompt:** Show me the details of SignalR @@ -7495,14 +7315,14 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| | 1 | 0.605571 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.427490 | `get_azure_messaging_service_details` | ❌ | +| 2 | 0.428387 | `get_azure_messaging_service_details` | ❌ | | 3 | 0.388473 | `get_azure_app_config_settings` | ❌ | | 4 | 0.379322 | `get_azure_app_resource_details` | ❌ | -| 5 | 0.322423 | `get_azure_data_explorer_kusto_details` | ❌ | +| 5 | 0.362474 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 415 +## Test 405 **Expected Tool:** `get_azure_signalr_details` **Prompt:** Show me the network information of SignalR runtime @@ -7511,15 +7331,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.639174 | `get_azure_signalr_details` | ✅ **EXPECTED** | -| 2 | 0.331031 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.321752 | `get_azure_app_resource_details` | ❌ | -| 4 | 0.317910 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.290120 | `get_azure_database_admin_configuration_details` | ❌ | +| 1 | 0.639126 | `get_azure_signalr_details` | ✅ **EXPECTED** | +| 2 | 0.332452 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.321748 | `get_azure_app_resource_details` | ❌ | +| 4 | 0.317889 | `get_azure_app_config_settings` | ❌ | +| 5 | 0.293598 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 416 +## Test 406 **Expected Tool:** `get_azure_confidential_ledger_entries` **Prompt:** Get entry from Confidential Ledger for transaction on ledger @@ -7532,11 +7352,11 @@ | 2 | 0.478164 | `append_azure_confidential_ledger_entries` | ❌ | | 3 | 0.212123 | `get_azure_key_vault_secret_values` | ❌ | | 4 | 0.171645 | `get_azure_app_config_settings` | ❌ | -| 5 | 0.141934 | `get_azure_database_admin_configuration_details` | ❌ | +| 5 | 0.141977 | `get_azure_database_admin_configuration_details` | ❌ | --- -## Test 417 +## Test 407 **Expected Tool:** `get_azure_confidential_ledger_entries` **Prompt:** Get transaction from ledger @@ -7548,12 +7368,12 @@ | 1 | 0.398313 | `get_azure_confidential_ledger_entries` | ✅ **EXPECTED** | | 2 | 0.394587 | `append_azure_confidential_ledger_entries` | ❌ | | 3 | 0.154670 | `get_azure_app_config_settings` | ❌ | -| 4 | 0.140497 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.137021 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.140490 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.137979 | `get_azure_cache_for_redis_details` | ❌ | --- -## Test 418 +## Test 408 **Expected Tool:** `append_azure_confidential_ledger_entries` **Prompt:** Append {"hello": "from mcp"} to my confidential ledger in collection @@ -7562,15 +7382,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521661 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | -| 2 | 0.358961 | `get_azure_confidential_ledger_entries` | ❌ | -| 3 | 0.217303 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.216503 | `update_azure_managed_lustre_filesystems` | ❌ | -| 5 | 0.204009 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.521636 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.358953 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.217335 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.216388 | `update_azure_managed_lustre_filesystems` | ❌ | +| 5 | 0.204048 | `edit_azure_app_config_settings` | ❌ | --- -## Test 419 +## Test 409 **Expected Tool:** `append_azure_confidential_ledger_entries` **Prompt:** Append an entry to my ledger with data {"key": "value"} @@ -7579,15 +7399,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.456236 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | -| 2 | 0.290162 | `get_azure_confidential_ledger_entries` | ❌ | -| 3 | 0.214449 | `create_azure_key_vault_secrets` | ❌ | -| 4 | 0.197784 | `create_azure_key_vault_items` | ❌ | -| 5 | 0.193367 | `import_azure_key_vault_certificates` | ❌ | +| 1 | 0.456250 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.289949 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.214352 | `create_azure_key_vault_secrets` | ❌ | +| 4 | 0.197816 | `create_azure_key_vault_items` | ❌ | +| 5 | 0.193278 | `import_azure_key_vault_certificates` | ❌ | --- -## Test 420 +## Test 410 **Expected Tool:** `append_azure_confidential_ledger_entries` **Prompt:** Create an immutable ledger entry in with content {"audit": "log"} @@ -7596,15 +7416,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.440219 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | -| 2 | 0.370010 | `get_azure_confidential_ledger_entries` | ❌ | -| 3 | 0.194427 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.185735 | `audit_azure_resources_compliance` | ❌ | -| 5 | 0.184506 | `create_azure_key_vault_items` | ❌ | +| 1 | 0.440329 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.370414 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.195062 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.186084 | `audit_azure_resources_compliance` | ❌ | +| 5 | 0.184154 | `create_azure_key_vault_items` | ❌ | --- -## Test 421 +## Test 411 **Expected Tool:** `append_azure_confidential_ledger_entries` **Prompt:** Write a tamper-proof entry to ledger containing {"transaction": "data"} @@ -7613,15 +7433,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.538122 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | -| 2 | 0.440073 | `get_azure_confidential_ledger_entries` | ❌ | -| 3 | 0.195454 | `edit_azure_app_config_settings` | ❌ | -| 4 | 0.181017 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.179716 | `create_azure_key_vault_secrets` | ❌ | +| 1 | 0.538340 | `append_azure_confidential_ledger_entries` | ✅ **EXPECTED** | +| 2 | 0.440278 | `get_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.195564 | `edit_azure_app_config_settings` | ❌ | +| 4 | 0.181246 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.179857 | `create_azure_key_vault_secrets` | ❌ | --- -## Test 422 +## Test 412 **Expected Tool:** `append_azure_confidential_ledger_entries` **Prompt:** Write an entry to confidential ledger @@ -7638,7 +7458,7 @@ --- -## Test 423 +## Test 413 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send an email from my communication service to @@ -7647,15 +7467,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.500055 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.199503 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.196433 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.186640 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.500057 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.197359 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.196910 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.186658 | `connect_azure_ai_foundry_agents` | ❌ | | 5 | 0.167947 | `recognize_speech_from_audio` | ❌ | --- -## Test 424 +## Test 414 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send an email to with subject @@ -7664,15 +7484,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.310898 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.159353 | `publish_azure_eventgrid_events` | ❌ | -| 3 | 0.126573 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.117898 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.112037 | `edit_azure_app_config_settings` | ❌ | +| 1 | 0.310846 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.159858 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.126443 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.117853 | `upload_azure_storage_blobs` | ❌ | +| 5 | 0.111964 | `edit_azure_app_config_settings` | ❌ | --- -## Test 425 +## Test 415 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send an email with BCC recipients @@ -7681,7 +7501,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.351224 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.351177 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.168924 | `append_azure_confidential_ledger_entries` | ❌ | | 3 | 0.162872 | `upload_azure_storage_blobs` | ❌ | | 4 | 0.156211 | `import_azure_key_vault_certificates` | ❌ | @@ -7689,7 +7509,7 @@ --- -## Test 426 +## Test 416 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send an SMS message to saying "Hello" @@ -7698,15 +7518,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.349463 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.349510 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.135823 | `generate_azure_cli_commands` | ❌ | | 3 | 0.127312 | `use_azure_openai_models` | ❌ | | 4 | 0.103749 | `lock_unlock_azure_app_config_settings` | ❌ | -| 5 | 0.101985 | `get_azure_messaging_service_details` | ❌ | +| 5 | 0.099666 | `get_azure_messaging_service_details` | ❌ | --- -## Test 427 +## Test 417 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send an SMS with delivery receipt tracking @@ -7715,15 +7535,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.553583 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.228899 | `append_azure_confidential_ledger_entries` | ❌ | -| 3 | 0.179812 | `get_azure_messaging_service_details` | ❌ | -| 4 | 0.167901 | `publish_azure_eventgrid_events` | ❌ | -| 5 | 0.166382 | `create_azure_load_testing` | ❌ | +| 1 | 0.553571 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.228969 | `append_azure_confidential_ledger_entries` | ❌ | +| 3 | 0.177229 | `get_azure_messaging_service_details` | ❌ | +| 4 | 0.168556 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.166446 | `create_azure_load_testing` | ❌ | --- -## Test 428 +## Test 418 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send broadcast SMS to and saying "Urgent notification" @@ -7732,15 +7552,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.368350 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.139528 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.120068 | `publish_azure_eventgrid_events` | ❌ | -| 4 | 0.118927 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.114379 | `lock_unlock_azure_app_config_settings` | ❌ | +| 1 | 0.368364 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.136766 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.120215 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.118743 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.114166 | `delete_azure_database_admin_configurations` | ❌ | --- -## Test 429 +## Test 419 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send email to multiple recipients: , @@ -7749,15 +7569,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.329704 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.087973 | `edit_azure_databases` | ❌ | +| 1 | 0.329715 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.087988 | `edit_azure_databases` | ❌ | | 3 | 0.083453 | `import_azure_key_vault_certificates` | ❌ | -| 4 | 0.077818 | `create_azure_workbooks` | ❌ | -| 5 | 0.076587 | `use_azure_openai_models` | ❌ | +| 4 | 0.077784 | `create_azure_workbooks` | ❌ | +| 5 | 0.076582 | `use_azure_openai_models` | ❌ | --- -## Test 430 +## Test 420 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send email with CC to and @@ -7766,15 +7586,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.344742 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.107004 | `import_azure_key_vault_certificates` | ❌ | -| 3 | 0.086249 | `create_azure_database_admin_configurations` | ❌ | -| 4 | 0.085420 | `append_azure_confidential_ledger_entries` | ❌ | -| 5 | 0.084922 | `upload_azure_storage_blobs` | ❌ | +| 1 | 0.344829 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.107081 | `import_azure_key_vault_certificates` | ❌ | +| 3 | 0.086260 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.085471 | `append_azure_confidential_ledger_entries` | ❌ | +| 5 | 0.084815 | `upload_azure_storage_blobs` | ❌ | --- -## Test 431 +## Test 421 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send email with custom sender name @@ -7783,15 +7603,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.267893 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.267887 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.159272 | `rename_azure_sql_databases` | ❌ | | 3 | 0.128130 | `edit_azure_databases` | ❌ | | 4 | 0.116966 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.112978 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.113303 | `publish_azure_eventgrid_events` | ❌ | --- -## Test 432 +## Test 422 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send email with reply-to address set to @@ -7800,7 +7620,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.258001 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.258019 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.145024 | `edit_azure_app_config_settings` | ❌ | | 3 | 0.139142 | `rename_azure_sql_databases` | ❌ | | 4 | 0.118904 | `edit_azure_databases` | ❌ | @@ -7808,7 +7628,7 @@ --- -## Test 433 +## Test 423 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send HTML-formatted email to with subject @@ -7817,15 +7637,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.240441 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.121727 | `publish_azure_eventgrid_events` | ❌ | -| 3 | 0.093090 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.092966 | `create_azure_monitor_webtests` | ❌ | -| 5 | 0.081317 | `use_azure_openai_models` | ❌ | +| 1 | 0.240446 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.122166 | `publish_azure_eventgrid_events` | ❌ | +| 3 | 0.093117 | `generate_azure_cli_commands` | ❌ | +| 4 | 0.092980 | `create_azure_monitor_webtests` | ❌ | +| 5 | 0.081360 | `use_azure_openai_models` | ❌ | --- -## Test 434 +## Test 424 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send SMS from my communication service to @@ -7834,15 +7654,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502800 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.193922 | `get_azure_messaging_service_details` | ❌ | -| 3 | 0.171065 | `connect_azure_ai_foundry_agents` | ❌ | +| 1 | 0.502877 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.190918 | `get_azure_messaging_service_details` | ❌ | +| 3 | 0.171099 | `connect_azure_ai_foundry_agents` | ❌ | | 4 | 0.160623 | `use_azure_openai_models` | ❌ | -| 5 | 0.141429 | `add_azure_app_service_database` | ❌ | +| 5 | 0.141456 | `add_azure_app_service_database` | ❌ | --- -## Test 435 +## Test 425 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send SMS message with custom tracking tag "campaign1" @@ -7851,7 +7671,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.450507 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.450517 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.149276 | `append_azure_confidential_ledger_entries` | ❌ | | 3 | 0.136341 | `use_azure_openai_models` | ❌ | | 4 | 0.134519 | `create_azure_load_testing` | ❌ | @@ -7859,7 +7679,7 @@ --- -## Test 436 +## Test 426 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send SMS to from with message "Test message" @@ -7868,7 +7688,7 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.397812 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.397926 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.160968 | `create_azure_load_testing` | ❌ | | 3 | 0.140980 | `rename_azure_sql_databases` | ❌ | | 4 | 0.136239 | `create_azure_monitor_webtests` | ❌ | @@ -7876,7 +7696,7 @@ --- -## Test 437 +## Test 427 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send SMS to multiple recipients: , @@ -7885,15 +7705,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.432791 | `send_azure_communication_messages` | ✅ **EXPECTED** | -| 2 | 0.107868 | `edit_azure_databases` | ❌ | -| 3 | 0.105406 | `use_azure_openai_models` | ❌ | -| 4 | 0.098377 | `create_azure_database_admin_configurations` | ❌ | -| 5 | 0.093660 | `install_azure_cli_extensions` | ❌ | +| 1 | 0.432998 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 2 | 0.107918 | `edit_azure_databases` | ❌ | +| 3 | 0.105595 | `use_azure_openai_models` | ❌ | +| 4 | 0.098489 | `create_azure_database_admin_configurations` | ❌ | +| 5 | 0.093663 | `install_azure_cli_extensions` | ❌ | --- -## Test 438 +## Test 428 **Expected Tool:** `send_azure_communication_messages` **Prompt:** Send SMS with delivery reporting enabled @@ -7902,15 +7722,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508960 | `send_azure_communication_messages` | ✅ **EXPECTED** | +| 1 | 0.508975 | `send_azure_communication_messages` | ✅ **EXPECTED** | | 2 | 0.198925 | `audit_azure_resources_compliance` | ❌ | | 3 | 0.182134 | `create_azure_monitor_webtests` | ❌ | -| 4 | 0.181769 | `get_azure_messaging_service_details` | ❌ | -| 5 | 0.180810 | `publish_azure_eventgrid_events` | ❌ | +| 4 | 0.180919 | `publish_azure_eventgrid_events` | ❌ | +| 5 | 0.179284 | `get_azure_messaging_service_details` | ❌ | --- -## Test 439 +## Test 429 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Convert speech to text from audio file using endpoint @@ -7922,12 +7742,12 @@ | 1 | 0.536383 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.242680 | `use_azure_openai_models` | ❌ | | 3 | 0.208042 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.186061 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.169609 | `upload_azure_storage_blobs` | ❌ | +| 4 | 0.194268 | `deploy_azure_ai_models` | ❌ | +| 5 | 0.186059 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | --- -## Test 440 +## Test 430 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Convert speech to text with comma-separated phrase hints: "Azure, cognitive services, API" @@ -7939,12 +7759,12 @@ | 1 | 0.527723 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.447401 | `use_azure_openai_models` | ❌ | | 3 | 0.438006 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.357169 | `send_azure_communication_messages` | ❌ | +| 4 | 0.357181 | `send_azure_communication_messages` | ❌ | | 5 | 0.327587 | `retrieve_azure_ai_knowledge_base_content` | ❌ | --- -## Test 441 +## Test 431 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Convert speech to text with detailed output format from audio file @@ -7953,15 +7773,15 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512611 | `recognize_speech_from_audio` | ✅ **EXPECTED** | -| 2 | 0.209831 | `generate_azure_cli_commands` | ❌ | -| 3 | 0.177099 | `use_azure_openai_models` | ❌ | -| 4 | 0.156153 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.133213 | `get_azure_ai_resources_details` | ❌ | +| 1 | 0.512680 | `recognize_speech_from_audio` | ✅ **EXPECTED** | +| 2 | 0.209836 | `generate_azure_cli_commands` | ❌ | +| 3 | 0.177107 | `use_azure_openai_models` | ❌ | +| 4 | 0.156184 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.133229 | `get_azure_ai_resources_details` | ❌ | --- -## Test 442 +## Test 432 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Convert this audio file to text using Azure Speech Services @@ -7974,11 +7794,11 @@ | 2 | 0.405604 | `use_azure_openai_models` | ❌ | | 3 | 0.393590 | `generate_azure_cli_commands` | ❌ | | 4 | 0.376904 | `upload_azure_storage_blobs` | ❌ | -| 5 | 0.371942 | `send_azure_communication_messages` | ❌ | +| 5 | 0.371891 | `send_azure_communication_messages` | ❌ | --- -## Test 443 +## Test 433 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Recognize speech from with phrase hints for better accuracy @@ -7990,12 +7810,12 @@ | 1 | 0.397874 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.257631 | `use_azure_openai_models` | ❌ | | 3 | 0.253810 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.217914 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.217932 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | | 5 | 0.183380 | `retrieve_azure_ai_knowledge_base_content` | ❌ | --- -## Test 444 +## Test 434 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Recognize speech from my audio file with language detection @@ -8006,13 +7826,13 @@ |------|-------|------|--------| | 1 | 0.493098 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.245003 | `use_azure_openai_models` | ❌ | -| 3 | 0.222843 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 4 | 0.172007 | `generate_azure_cli_commands` | ❌ | -| 5 | 0.165060 | `evaluate_azure_ai_foundry_agents` | ❌ | +| 3 | 0.222850 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 4 | 0.183681 | `deploy_azure_ai_models` | ❌ | +| 5 | 0.172007 | `generate_azure_cli_commands` | ❌ | --- -## Test 445 +## Test 435 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Transcribe audio using multiple phrase hints: "Azure", "cognitive services", "machine learning" @@ -8024,12 +7844,12 @@ | 1 | 0.539450 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.453170 | `generate_azure_cli_commands` | ❌ | | 3 | 0.451471 | `use_azure_openai_models` | ❌ | -| 4 | 0.361207 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.346335 | `get_azure_ai_resources_details` | ❌ | +| 4 | 0.361161 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.348465 | `deploy_azure_ai_models` | ❌ | --- -## Test 446 +## Test 436 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Transcribe audio with raw profanity output from file @@ -8041,12 +7861,12 @@ | 1 | 0.359302 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.194397 | `generate_azure_cli_commands` | ❌ | | 3 | 0.170664 | `use_azure_openai_models` | ❌ | -| 4 | 0.146733 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.132184 | `get_azure_key_vault_secret_values` | ❌ | +| 4 | 0.146774 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.134187 | `deploy_azure_ai_models` | ❌ | --- -## Test 447 +## Test 437 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Transcribe speech from audio file with profanity filtering @@ -8058,12 +7878,12 @@ | 1 | 0.380445 | `recognize_speech_from_audio` | ✅ **EXPECTED** | | 2 | 0.188454 | `use_azure_openai_models` | ❌ | | 3 | 0.170544 | `generate_azure_cli_commands` | ❌ | -| 4 | 0.161213 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | -| 5 | 0.116660 | `create_azure_database_admin_configurations` | ❌ | +| 4 | 0.161234 | `query_and_evaluate_azure_ai_foundry_agents` | ❌ | +| 5 | 0.124053 | `deploy_azure_ai_models` | ❌ | --- -## Test 448 +## Test 438 **Expected Tool:** `recognize_speech_from_audio` **Prompt:** Transcribe the audio file in Spanish language @@ -8082,29 +7902,29 @@ ## Summary -**Total Prompts Tested:** 448 -**Analysis Execution Time:** 124.3936740s +**Total Prompts Tested:** 438 +**Analysis Execution Time:** 85.3316282s ### Success Rate Metrics -**Top Choice Success:** 77.5% (347/448 tests) +**Top Choice Success:** 76.3% (334/438 tests) #### Confidence Level Distribution -**💪 Very High Confidence (≥0.8):** 0.0% (0/448 tests) -**🎯 High Confidence (≥0.7):** 3.1% (14/448 tests) -**✅ Good Confidence (≥0.6):** 17.9% (80/448 tests) -**👍 Fair Confidence (≥0.5):** 59.4% (266/448 tests) -**👌 Acceptable Confidence (≥0.4):** 87.1% (390/448 tests) -**❌ Low Confidence (<0.4):** 12.9% (58/448 tests) +**💪 Very High Confidence (≥0.8):** 0.0% (0/438 tests) +**🎯 High Confidence (≥0.7):** 3.2% (14/438 tests) +**✅ Good Confidence (≥0.6):** 18.0% (79/438 tests) +**👍 Fair Confidence (≥0.5):** 59.4% (260/438 tests) +**👌 Acceptable Confidence (≥0.4):** 87.0% (381/438 tests) +**❌ Low Confidence (<0.4):** 13.0% (57/438 tests) #### Top Choice + Confidence Combinations -**💪 Top Choice + Very High Confidence (≥0.8):** 0.0% (0/448 tests) -**🎯 Top Choice + High Confidence (≥0.7):** 3.1% (14/448 tests) -**✅ Top Choice + Good Confidence (≥0.6):** 17.0% (76/448 tests) -**👍 Top Choice + Fair Confidence (≥0.5):** 50.9% (228/448 tests) -**👌 Top Choice + Acceptable Confidence (≥0.4):** 69.0% (309/448 tests) +**💪 Top Choice + Very High Confidence (≥0.8):** 0.0% (0/438 tests) +**🎯 Top Choice + High Confidence (≥0.7):** 3.2% (14/438 tests) +**✅ Top Choice + Good Confidence (≥0.6):** 17.1% (75/438 tests) +**👍 Top Choice + Fair Confidence (≥0.5):** 50.0% (219/438 tests) +**👌 Top Choice + Acceptable Confidence (≥0.4):** 67.8% (297/438 tests) ### Success Rate Analysis diff --git a/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs b/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs index 4622a8b0f6..b82ce93524 100644 --- a/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs +++ b/tools/Azure.Mcp.Tools.Communication/src/Commands/Email/EmailSendCommand.cs @@ -40,7 +40,7 @@ Supports HTML content and CC/BCC recipients. Destructive = false, ReadOnly = true, OpenWorld = true, - Idempotent = true, + Idempotent = false, Secret = false, LocalRequired = false }; diff --git a/tools/Azure.Mcp.Tools.Communication/src/Commands/Sms/SmsSendCommand.cs b/tools/Azure.Mcp.Tools.Communication/src/Commands/Sms/SmsSendCommand.cs index 8a11fa1857..be1a297434 100644 --- a/tools/Azure.Mcp.Tools.Communication/src/Commands/Sms/SmsSendCommand.cs +++ b/tools/Azure.Mcp.Tools.Communication/src/Commands/Sms/SmsSendCommand.cs @@ -36,7 +36,7 @@ public sealed class SmsSendCommand(ILogger logger) : BaseCommuni Destructive = false, ReadOnly = true, OpenWorld = true, - Idempotent = true, + Idempotent = false, Secret = false, LocalRequired = false };