Add list custom LLM Obs evaluator configs endpoint

.generator/schemas/v2/openapi.yaml

@@ -52338,6 +52338,10 @@ components:
     LLMObsCustomEvalConfigBedrockOptions:
       description: AWS Bedrock-specific options for LLM provider configuration.
       properties:
+        inference_profile:
+          description: Bedrock inference profile identifier, such as an application inference profile ARN.
+          example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"
+          type: string
         region:
           description: AWS region for Bedrock.
           example: "us-east-1"
@@ -52428,6 +52432,11 @@ components:
       properties:
         assessment_criteria:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria"
+        context_query:
+          description: Query used to extract additional context for the evaluation.
+          example: "@input.context"
+          nullable: true
+          type: string
         inference_params:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams"
         last_used_library_prompt_template_name:
@@ -52452,6 +52461,15 @@ components:
           items:
             $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
           type: array
+        target_query:
+          description: Query used to extract the target value to evaluate.
+          example: "@output.value"
+          nullable: true
+          type: string
+        user_specified_json_post_processing_function:
+          description: User-provided function applied to post-process the JSON output of the LLM judge.
+          nullable: true
+          type: string
       required:
         - inference_params
       type: object
@@ -52473,16 +52491,29 @@ components:
         vertex_ai:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions"
       type: object
+    LLMObsCustomEvalConfigListResponse:
+      description: Response containing a list of custom LLM Observability evaluator configurations.
+      properties:
+        data:
+          description: List of custom evaluator configuration data objects.
+          items:
+            $ref: "#/components/schemas/LLMObsCustomEvalConfigData"
+          type: array
+      required:
+        - data
+      type: object
     LLMObsCustomEvalConfigParsingType:
       description: Output parsing type for a custom LLM judge evaluator.
       enum:
         - structured_output
         - json
+        - keyword_search
       example: "structured_output"
       type: string
       x-enum-varnames:
         - STRUCTURED_OUTPUT
         - JSON
+        - KEYWORD_SEARCH
     LLMObsCustomEvalConfigPromptContent:
       description: A content block within a prompt message.
       properties:
@@ -52589,6 +52620,13 @@ components:
         eval_scope:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope"
           nullable: true
+        experiment_project_ids:
+          description: Experiment project IDs this evaluator is scoped to.
+          items:
+            description: An experiment project ID.
+            format: uuid
+            type: string
+          type: array
         filter:
           description: Filter expression to select which spans to evaluate.
           example: "@service:my-service"
@@ -116425,6 +116463,67 @@ paths:
       x-unstable: |-
         This endpoint is in Preview and may introduce breaking changes.
         If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
+  /api/unstable/llm-obs/config/evaluators/custom:
+    get:
+      description: List all custom LLM Observability evaluator configurations for the organization.
+      operationId: ListLLMObsCustomEvalConfigs
+      responses:
+        "200":
+          content:
+            application/json:
+              examples:
+                default:
+                  value:
+                    data:
+                      - attributes:
+                          category: "Custom"
+                          created_at: "2024-01-15T10:30:00Z"
+                          created_by:
+                            email: "user@example.com"
+                          eval_name: "my-custom-evaluator"
+                          last_updated_by:
+                            email: "user@example.com"
+                          llm_judge_config:
+                            inference_params:
+                              max_tokens: 1024
+                              temperature: 0.7
+                            parsing_type: "structured_output"
+                          llm_provider:
+                            integration_provider: "openai"
+                            model_name: "gpt-4o"
+                          target:
+                            application_name: "my-llm-app"
+                            enabled: true
+                            sampling_percentage: 50.0
+                          updated_at: "2024-01-15T10:30:00Z"
+                        id: "my-custom-evaluator"
+                        type: "evaluator_config"
+              schema:
+                $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse"
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Forbidden
+        "429":
+          $ref: "#/components/responses/TooManyRequestsResponse"
+      security:
+        - apiKeyAuth: []
+          appKeyAuth: []
+      summary: List custom evaluator configurations
+      tags:
+        - LLM Observability
+      x-unstable: |-
+        **Note**: This endpoint is in preview and is subject to change.
+        If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
   /api/unstable/llm-obs/config/evaluators/custom/{eval_name}:
     delete:
       description: Delete a custom LLM Observability evaluator configuration by its name.

examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.java

@@ -0,0 +1,25 @@
+// List custom evaluator configurations returns "OK" response
+
+import com.datadog.api.client.ApiClient;
+import com.datadog.api.client.ApiException;
+import com.datadog.api.client.v2.api.LlmObservabilityApi;
+import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigListResponse;
+
+public class Example {
+  public static void main(String[] args) {
+    ApiClient defaultClient = ApiClient.getDefaultApiClient();
+    defaultClient.setUnstableOperationEnabled("v2.listLLMObsCustomEvalConfigs", true);
+    LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
+
+    try {
+      LLMObsCustomEvalConfigListResponse result = apiInstance.listLLMObsCustomEvalConfigs();
+      System.out.println(result);
+    } catch (ApiException e) {
+      System.err.println("Exception when calling LlmObservabilityApi#listLLMObsCustomEvalConfigs");
+      System.err.println("Status code: " + e.getCode());
+      System.err.println("Reason: " + e.getResponseBody());
+      System.err.println("Response headers: " + e.getResponseHeaders());
+      e.printStackTrace();
+    }
+  }
+}

examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java

@@ -47,6 +47,7 @@ public static void main(String[] args) {
                                             .minThreshold(0.7)
                                             .passValues(Arrays.asList("pass", "yes"))
                                             .passWhen(true))
+                                    .contextQuery("@input.context")
                                     .inferenceParams(
                                         new LLMObsCustomEvalConfigInferenceParams()
                                             .maxTokens(1024L)
@@ -87,11 +88,15 @@ public static void main(String[] args) {
                                                                             .result("sunny, 72F")
                                                                             .toolId("call_abc123")
                                                                             .type("function")))))
-                                                .role("user"))))
+                                                .role("user")))
+                                    .targetQuery("@output.value")
+                                    .userSpecifiedJsonPostProcessingFunction(null))
                             .llmProvider(
                                 new LLMObsCustomEvalConfigLLMProvider()
                                     .bedrock(
                                         new LLMObsCustomEvalConfigBedrockOptions()
+                                            .inferenceProfile(
+                                                "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123")
                                             .region("us-east-1"))
                                     .integrationAccountId("my-account-id")
                                     .integrationProvider(

src/main/java/com/datadog/api/client/ApiClient.java

@@ -812,6 +812,7 @@ public class ApiClient {
           put("v2.getLLMObsPrompt", false);
           put("v2.getLLMObsPromptVersion", false);
           put("v2.listLLMObsAnnotationQueues", false);
+          put("v2.listLLMObsCustomEvalConfigs", false);
           put("v2.listLLMObsDatasetRecords", false);
           put("v2.listLLMObsDatasets", false);
           put("v2.listLLMObsDatasetVersions", false);

src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java

@@ -18,6 +18,7 @@
 import com.datadog.api.client.v2.model.LLMObsAnnotationsResponse;
 import com.datadog.api.client.v2.model.LLMObsCreatePromptRequest;
 import com.datadog.api.client.v2.model.LLMObsCreatePromptVersionRequest;
+import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigListResponse;
 import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigResponse;
 import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigUpdateRequest;
 import com.datadog.api.client.v2.model.LLMObsDataDeletionRequest;
@@ -6391,6 +6392,137 @@ public ApiResponse<LLMObsAnnotationQueuesResponse> listLLMObsAnnotationQueuesWit
         new GenericType<LLMObsAnnotationQueuesResponse>() {});
   }
 
+  /**
+   * List custom evaluator configurations.
+   *
+   * <p>See {@link #listLLMObsCustomEvalConfigsWithHttpInfo}.
+   *
+   * @return LLMObsCustomEvalConfigListResponse
+   * @throws ApiException if fails to make API call
+   */
+  public LLMObsCustomEvalConfigListResponse listLLMObsCustomEvalConfigs() throws ApiException {
+    return listLLMObsCustomEvalConfigsWithHttpInfo().getData();
+  }
+
+  /**
+   * List custom evaluator configurations.
+   *
+   * <p>See {@link #listLLMObsCustomEvalConfigsWithHttpInfoAsync}.
+   *
+   * @return CompletableFuture&lt;LLMObsCustomEvalConfigListResponse&gt;
+   */
+  public CompletableFuture<LLMObsCustomEvalConfigListResponse> listLLMObsCustomEvalConfigsAsync() {
+    return listLLMObsCustomEvalConfigsWithHttpInfoAsync()
+        .thenApply(
+            response -> {
+              return response.getData();
+            });
+  }
+
+  /**
+   * List all custom LLM Observability evaluator configurations for the organization.
+   *
+   * @return ApiResponse&lt;LLMObsCustomEvalConfigListResponse&gt;
+   * @throws ApiException if fails to make API call
+   * @http.response.details
+   *     <table border="1">
+   *    <caption>Response details</caption>
+   *       <tr><td> Status Code </td><td> Description </td><td> Response Headers </td></tr>
+   *       <tr><td> 200 </td><td> OK </td><td>  -  </td></tr>
+   *       <tr><td> 401 </td><td> Unauthorized </td><td>  -  </td></tr>
+   *       <tr><td> 403 </td><td> Forbidden </td><td>  -  </td></tr>
+   *       <tr><td> 429 </td><td> Too many requests </td><td>  -  </td></tr>
+   *     </table>
+   */
+  public ApiResponse<LLMObsCustomEvalConfigListResponse> listLLMObsCustomEvalConfigsWithHttpInfo()
+      throws ApiException {
+    // Check if unstable operation is enabled
+    String operationId = "listLLMObsCustomEvalConfigs";
+    if (apiClient.isUnstableOperationEnabled("v2." + operationId)) {
+      apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId));
+    } else {
+      throw new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId));
+    }
+    Object localVarPostBody = null;
+    // create path and map variables
+    String localVarPath = "/api/unstable/llm-obs/config/evaluators/custom";
+
+    Map<String, String> localVarHeaderParams = new HashMap<String, String>();
+
+    Invocation.Builder builder =
+        apiClient.createBuilder(
+            "v2.LlmObservabilityApi.listLLMObsCustomEvalConfigs",
+            localVarPath,
+            new ArrayList<Pair>(),
+            localVarHeaderParams,
+            new HashMap<String, String>(),
+            new String[] {"application/json"},
+            new String[] {"apiKeyAuth", "appKeyAuth"});
+    return apiClient.invokeAPI(
+        "GET",
+        builder,
+        localVarHeaderParams,
+        new String[] {},
+        localVarPostBody,
+        new HashMap<String, Object>(),
+        false,
+        new GenericType<LLMObsCustomEvalConfigListResponse>() {});
+  }
+
+  /**
+   * List custom evaluator configurations.
+   *
+   * <p>See {@link #listLLMObsCustomEvalConfigsWithHttpInfo}.
+   *
+   * @return CompletableFuture&lt;ApiResponse&lt;LLMObsCustomEvalConfigListResponse&gt;&gt;
+   */
+  public CompletableFuture<ApiResponse<LLMObsCustomEvalConfigListResponse>>
+      listLLMObsCustomEvalConfigsWithHttpInfoAsync() {
+    // Check if unstable operation is enabled
+    String operationId = "listLLMObsCustomEvalConfigs";
+    if (apiClient.isUnstableOperationEnabled("v2." + operationId)) {
+      apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId));
+    } else {
+      CompletableFuture<ApiResponse<LLMObsCustomEvalConfigListResponse>> result =
+          new CompletableFuture<>();
+      result.completeExceptionally(
+          new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId)));
+      return result;
+    }
+    Object localVarPostBody = null;
+    // create path and map variables
+    String localVarPath = "/api/unstable/llm-obs/config/evaluators/custom";
+
+    Map<String, String> localVarHeaderParams = new HashMap<String, String>();
+
+    Invocation.Builder builder;
+    try {
+      builder =
+          apiClient.createBuilder(
+              "v2.LlmObservabilityApi.listLLMObsCustomEvalConfigs",
+              localVarPath,
+              new ArrayList<Pair>(),
+              localVarHeaderParams,
+              new HashMap<String, String>(),
+              new String[] {"application/json"},
+              new String[] {"apiKeyAuth", "appKeyAuth"});
+    } catch (ApiException ex) {
+      CompletableFuture<ApiResponse<LLMObsCustomEvalConfigListResponse>> result =
+          new CompletableFuture<>();
+      result.completeExceptionally(ex);
+      return result;
+    }
+    return apiClient.invokeAPIAsync(
+        "GET",
+        builder,
+        localVarHeaderParams,
+        new String[] {},
+        localVarPostBody,
+        new HashMap<String, Object>(),
+        false,
+        new GenericType<LLMObsCustomEvalConfigListResponse>() {});
+  }
+
   /** Manage optional parameters to listLLMObsDatasetRecords. */
   public static class ListLLMObsDatasetRecordsOptionalParameters {
     private Long filterVersion;