diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts new file mode 100644 index 00000000000000..0a6281d69d1096 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Get Evaluate API endpoint + * version: 1 + */ + +export type GetEvaluateResponse = z.infer; +export const GetEvaluateResponse = z.object({ + agentExecutors: z.array(z.string()), +}); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml new file mode 100644 index 00000000000000..b0c0c218eb9ac3 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml @@ -0,0 +1,40 @@ +openapi: 3.0.0 +info: + title: Get Evaluate API endpoint + version: '1' +paths: + /internal/elastic_assistant/evaluate: + get: + operationId: GetEvaluate + x-codegen-enabled: true + description: Get relevant data for performing an evaluation like available sample data, agents, and evaluators + summary: Get relevant data for performing an evaluation + tags: + - Evaluation API + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + agentExecutors: + type: array + items: + type: string + required: + - agentExecutors + '400': + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts new file mode 100644 index 00000000000000..d5d1177a9c16e1 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Post Evaluate API endpoint + * version: 1 + */ + +export type OutputIndex = z.infer; +export const OutputIndex = z.string().regex(/^.kibana-elastic-ai-assistant-/); + +export type DatasetItem = z.infer; +export const DatasetItem = z.object({ + id: z.string().optional(), + input: z.string(), + prediction: z.string().optional(), + reference: z.string(), + tags: z.array(z.string()).optional(), +}); + +export type Dataset = z.infer; +export const Dataset = z.array(DatasetItem).default([]); + +export type PostEvaluateBody = z.infer; +export const PostEvaluateBody = z.object({ + dataset: Dataset.optional(), + evalPrompt: z.string().optional(), +}); + +export type PostEvaluateRequestQuery = z.infer; +export const PostEvaluateRequestQuery = z.object({ + /** + * Agents parameter description + */ + agents: z.string(), + /** + * Dataset Name parameter description + */ + datasetName: z.string().optional(), + /** + * Evaluation Type parameter description + */ + evaluationType: z.string().optional(), + /** + * Eval Model parameter description + */ + evalModel: z.string().optional(), + /** + * Models parameter description + */ + models: z.string(), + /** + * Output Index parameter description + */ + outputIndex: OutputIndex, + /** + * Project Name parameter description + */ + projectName: z.string().optional(), + /** + * Run Name parameter description + */ + runName: z.string().optional(), +}); +export type PostEvaluateRequestQueryInput = z.input; + +export type PostEvaluateRequestBody = z.infer; +export const PostEvaluateRequestBody = PostEvaluateBody; +export type PostEvaluateRequestBodyInput = z.input; + +export type PostEvaluateResponse = z.infer; +export const PostEvaluateResponse = z.object({ + evaluationId: z.string(), + success: z.boolean(), +}); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml new file mode 100644 index 00000000000000..41a7230e85ac5a --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml @@ -0,0 +1,126 @@ +openapi: 3.0.0 +info: + title: Post Evaluate API endpoint + version: '1' +paths: + /internal/elastic_assistant/evaluate: + post: + operationId: PostEvaluate + x-codegen-enabled: true + description: Perform an evaluation using sample data against a combination of Agents and Connectors + summary: Performs an evaluation of the Elastic Assistant + tags: + - Evaluation API + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PostEvaluateBody' + parameters: + - name: agents + in: query + description: Agents parameter description + required: true + schema: + type: string + - name: datasetName + in: query + description: Dataset Name parameter description + schema: + type: string + - name: evaluationType + in: query + description: Evaluation Type parameter description + schema: + type: string + - name: evalModel + in: query + description: Eval Model parameter description + schema: + type: string + - name: models + in: query + description: Models parameter description + required: true + schema: + type: string + - name: outputIndex + in: query + description: Output Index parameter description + required: true + schema: + $ref: '#/components/schemas/OutputIndex' + - name: projectName + in: query + description: Project Name parameter description + schema: + type: string + - name: runName + in: query + description: Run Name parameter description + schema: + type: string + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + evaluationId: + type: string + success: + type: boolean + required: + - evaluationId + - success + '400': + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string +components: + schemas: + OutputIndex: + type: string + pattern: '^.kibana-elastic-ai-assistant-' + DatasetItem: + type: object + properties: + id: + type: string + input: + type: string + prediction: + type: string + reference: + type: string + tags: + type: array + items: + type: string + required: + - input + - reference + Dataset: + type: array + items: + $ref: '#/components/schemas/DatasetItem' + default: [] + PostEvaluateBody: + type: object + properties: + dataset: + $ref: '#/components/schemas/Dataset' + evalPrompt: + type: string diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts new file mode 100644 index 00000000000000..4257cb9bae149b --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// API versioning constants +export const API_VERSIONS = { + public: { + v1: '2023-10-31', + }, + internal: { + v1: '1', + }, +}; + +export const PUBLIC_API_ACCESS = 'public'; +export const INTERNAL_API_ACCESS = 'internal'; + +// Evaluation Schemas +export * from './evaluation/post_evaluate_route.gen'; +export * from './evaluation/get_evaluate_route.gen'; + +// Capabilities Schemas +export * from './capabilities/get_capabilities_route.gen'; diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts index a2576038c6f51a..e285be395c71cd 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts @@ -5,7 +5,8 @@ * 2.0. */ -export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen'; +// Schema constants +export * from './impl/schemas'; export { defaultAssistantFeatures } from './impl/capabilities'; export type { AssistantFeatures } from './impl/capabilities'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx index 4c71c1e63f8b3e..26a37e12c4e539 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx @@ -13,7 +13,6 @@ import { fetchConnectorExecuteAction, FetchConnectorExecuteAction, getKnowledgeBaseStatus, - postEvaluation, postKnowledgeBase, } from './api'; import type { Conversation, Message } from '../assistant_context/types'; @@ -340,52 +339,4 @@ describe('API tests', () => { await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); }); }); - - describe('postEvaluation', () => { - it('calls the knowledge base API when correct resource path', async () => { - (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true }); - const testProps = { - http: mockHttp, - evalParams: { - agents: ['not', 'alphabetical'], - dataset: '{}', - datasetName: 'Test Dataset', - projectName: 'Test Project Name', - runName: 'Test Run Name', - evalModel: ['not', 'alphabetical'], - evalPrompt: 'evalPrompt', - evaluationType: ['not', 'alphabetical'], - models: ['not', 'alphabetical'], - outputIndex: 'outputIndex', - }, - }; - - await postEvaluation(testProps); - - expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { - method: 'POST', - body: '{"dataset":{},"evalPrompt":"evalPrompt"}', - headers: { 'Content-Type': 'application/json' }, - query: { - models: 'alphabetical,not', - agents: 'alphabetical,not', - datasetName: 'Test Dataset', - evaluationType: 'alphabetical,not', - evalModel: 'alphabetical,not', - outputIndex: 'outputIndex', - projectName: 'Test Project Name', - runName: 'Test Run Name', - }, - signal: undefined, - }); - }); - it('returns error when error is an error', async () => { - const error = 'simulated error'; - (mockHttp.fetch as jest.Mock).mockImplementation(() => { - throw new Error(error); - }); - - await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); - }); - }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index f04b99c4e46e19..c18193c7fa0a6a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -16,7 +16,6 @@ import { getOptionalRequestParams, hasParsableResponse, } from './helpers'; -import { PerformEvaluationParams } from './settings/evaluation_settings/use_perform_evaluation'; export interface FetchConnectorExecuteAction { isEnabledRAGAlerts: boolean; @@ -335,61 +334,3 @@ export const deleteKnowledgeBase = async ({ return error as IHttpFetchError; } }; - -export interface PostEvaluationParams { - http: HttpSetup; - evalParams?: PerformEvaluationParams; - signal?: AbortSignal | undefined; -} - -export interface PostEvaluationResponse { - evaluationId: string; - success: boolean; -} - -/** - * API call for evaluating models. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.evalParams] - Params necessary for evaluation - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const postEvaluation = async ({ - http, - evalParams, - signal, -}: PostEvaluationParams): Promise => { - try { - const path = `/internal/elastic_assistant/evaluate`; - const query = { - agents: evalParams?.agents.sort()?.join(','), - datasetName: evalParams?.datasetName, - evaluationType: evalParams?.evaluationType.sort()?.join(','), - evalModel: evalParams?.evalModel.sort()?.join(','), - outputIndex: evalParams?.outputIndex, - models: evalParams?.models.sort()?.join(','), - projectName: evalParams?.projectName, - runName: evalParams?.runName, - }; - - const response = await http.fetch(path, { - method: 'POST', - body: JSON.stringify({ - dataset: JSON.parse(evalParams?.dataset ?? '[]'), - evalPrompt: evalParams?.evalPrompt ?? '', - }), - headers: { - 'Content-Type': 'application/json', - }, - query, - signal, - }); - - return response as PostEvaluationResponse; - } catch (error) { - return error as IHttpFetchError; - } -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx index b41d7ac1445549..30c113eb0e8036 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx @@ -13,7 +13,7 @@ import { API_ERROR } from '../../translations'; jest.mock('@kbn/core-http-browser'); const mockHttp = { - fetch: jest.fn(), + get: jest.fn(), } as unknown as HttpSetup; describe('Capabilities API tests', () => { @@ -25,15 +25,14 @@ describe('Capabilities API tests', () => { it('calls the internal assistant API for fetching assistant capabilities', async () => { await getCapabilities({ http: mockHttp }); - expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { - method: 'GET', + expect(mockHttp.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { signal: undefined, version: '1', }); }); it('returns API_ERROR when the response status is error', async () => { - (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: API_ERROR }); + (mockHttp.get as jest.Mock).mockResolvedValue({ status: API_ERROR }); const result = await getCapabilities({ http: mockHttp }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx index 59927dbf2c4721..96e6660f6bc0ef 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx @@ -6,7 +6,7 @@ */ import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; -import { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; +import { API_VERSIONS, GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; export interface GetCapabilitiesParams { http: HttpSetup; @@ -29,13 +29,10 @@ export const getCapabilities = async ({ try { const path = `/internal/elastic_assistant/capabilities`; - const response = await http.fetch(path, { - method: 'GET', + return await http.get(path, { signal, - version: '1', + version: API_VERSIONS.internal.v1, }); - - return response as GetCapabilitiesResponse; } catch (error) { return error as IHttpFetchError; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx index c9e60b806d1bf3..b7648983e6f7ad 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx @@ -11,11 +11,12 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import type { ReactNode } from 'react'; import React from 'react'; import { useCapabilities, UseCapabilitiesParams } from './use_capabilities'; +import { API_VERSIONS } from '@kbn/elastic-assistant-common'; const statusResponse = { assistantModelEvaluation: true, assistantStreamingEnabled: false }; const http = { - fetch: jest.fn().mockResolvedValue(statusResponse), + get: jest.fn().mockResolvedValue(statusResponse), }; const toasts = { addError: jest.fn(), @@ -36,14 +37,10 @@ describe('useFetchRelatedCases', () => { wrapper: createWrapper(), }); - expect(defaultProps.http.fetch).toHaveBeenCalledWith( - '/internal/elastic_assistant/capabilities', - { - method: 'GET', - version: '1', - signal: new AbortController().signal, - } - ); + expect(defaultProps.http.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { + version: API_VERSIONS.internal.v1, + signal: new AbortController().signal, + }); expect(toasts.addError).not.toHaveBeenCalled(); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx new file mode 100644 index 00000000000000..d25953370e97ad --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { postEvaluation } from './evaluate'; +import { HttpSetup } from '@kbn/core-http-browser'; +import { API_VERSIONS } from '@kbn/elastic-assistant-common'; + +jest.mock('@kbn/core-http-browser'); + +const mockHttp = { + post: jest.fn(), +} as unknown as HttpSetup; + +describe('postEvaluation', () => { + it('calls the knowledge base API when correct resource path', async () => { + (mockHttp.post as jest.Mock).mockResolvedValue({ success: true }); + const testProps = { + http: mockHttp, + evalParams: { + agents: ['not', 'alphabetical'], + dataset: '{}', + datasetName: 'Test Dataset', + projectName: 'Test Project Name', + runName: 'Test Run Name', + evalModel: ['not', 'alphabetical'], + evalPrompt: 'evalPrompt', + evaluationType: ['not', 'alphabetical'], + models: ['not', 'alphabetical'], + outputIndex: 'outputIndex', + }, + }; + + await postEvaluation(testProps); + + expect(mockHttp.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + body: '{"dataset":{},"evalPrompt":"evalPrompt"}', + headers: { 'Content-Type': 'application/json' }, + query: { + models: 'alphabetical,not', + agents: 'alphabetical,not', + datasetName: 'Test Dataset', + evaluationType: 'alphabetical,not', + evalModel: 'alphabetical,not', + outputIndex: 'outputIndex', + projectName: 'Test Project Name', + runName: 'Test Run Name', + }, + signal: undefined, + version: API_VERSIONS.internal.v1, + }); + }); + it('returns error when error is an error', async () => { + const error = 'simulated error'; + (mockHttp.post as jest.Mock).mockImplementation(() => { + throw new Error(error); + }); + + const knowledgeBaseArgs = { + resource: 'a-resource', + http: mockHttp, + }; + + await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); + }); +}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx new file mode 100644 index 00000000000000..6581e22e77921a --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; +import { + API_VERSIONS, + GetEvaluateResponse, + PostEvaluateResponse, +} from '@kbn/elastic-assistant-common'; +import { PerformEvaluationParams } from './use_perform_evaluation'; + +export interface PostEvaluationParams { + http: HttpSetup; + evalParams?: PerformEvaluationParams; + signal?: AbortSignal | undefined; +} + +/** + * API call for evaluating models. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.evalParams] - Params necessary for evaluation + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const postEvaluation = async ({ + http, + evalParams, + signal, +}: PostEvaluationParams): Promise => { + try { + const path = `/internal/elastic_assistant/evaluate`; + const query = { + agents: evalParams?.agents.sort()?.join(','), + datasetName: evalParams?.datasetName, + evaluationType: evalParams?.evaluationType.sort()?.join(','), + evalModel: evalParams?.evalModel.sort()?.join(','), + outputIndex: evalParams?.outputIndex, + models: evalParams?.models.sort()?.join(','), + projectName: evalParams?.projectName, + runName: evalParams?.runName, + }; + + return await http.post(path, { + body: JSON.stringify({ + dataset: JSON.parse(evalParams?.dataset ?? '[]'), + evalPrompt: evalParams?.evalPrompt ?? '', + }), + headers: { + 'Content-Type': 'application/json', + }, + query, + signal, + version: API_VERSIONS.internal.v1, + }); + } catch (error) { + return error as IHttpFetchError; + } +}; + +export interface GetEvaluationParams { + http: HttpSetup; + signal?: AbortSignal | undefined; +} + +/** + * API call for fetching evaluation data. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const getEvaluation = async ({ + http, + signal, +}: GetEvaluationParams): Promise => { + try { + const path = `/internal/elastic_assistant/evaluate`; + + return await http.get(path, { + signal, + version: API_VERSIONS.internal.v1, + }); + } catch (error) { + return error as IHttpFetchError; + } +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx new file mode 100644 index 00000000000000..a37cf18a235ecd --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from '@tanstack/react-query'; +import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; +import type { IToasts } from '@kbn/core-notifications-browser'; +import { i18n } from '@kbn/i18n'; +import { getEvaluation } from './evaluate'; + +const EVALUATION_DATA_QUERY_KEY = ['elastic-assistant', 'evaluation-data']; + +export interface UseEvaluationDataParams { + http: HttpSetup; + toasts?: IToasts; +} + +/** + * Hook for fetching evaluation data, like available agents, test data, etc + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {IToasts} [options.toasts] - IToasts + * + * @returns {useMutation} mutation hook for setting up the Knowledge Base + */ +export const useEvaluationData = ({ http, toasts }: UseEvaluationDataParams) => { + return useQuery({ + queryKey: EVALUATION_DATA_QUERY_KEY, + queryFn: ({ signal }) => { + // Optional params workaround: see: https://github.com/TanStack/query/issues/1077#issuecomment-1431247266 + return getEvaluation({ http, signal }); + }, + retry: false, + keepPreviousData: true, + // Deprecated, hoist to `queryCache` w/in `QueryClient. See: https://stackoverflow.com/a/76961109 + onError: (error: IHttpFetchError) => { + if (error.name !== 'AbortError') { + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.evaluation.fetchEvaluationDataError', { + defaultMessage: 'Error fetching evaluation data...', + }), + }); + } + }, + }); +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx similarity index 81% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx index b0653384805497..f9fdb2e80b7b2d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx @@ -7,14 +7,15 @@ import { act, renderHook } from '@testing-library/react-hooks'; import { usePerformEvaluation, UsePerformEvaluationParams } from './use_perform_evaluation'; -import { postEvaluation as _postEvaluation } from '../../api'; +import { postEvaluation as _postEvaluation } from './evaluate'; import { useMutation as _useMutation } from '@tanstack/react-query'; +import { API_VERSIONS } from '@kbn/elastic-assistant-common'; const useMutationMock = _useMutation as jest.Mock; const postEvaluationMock = _postEvaluation as jest.Mock; -jest.mock('../../api', () => { - const actual = jest.requireActual('../../api'); +jest.mock('./evaluate', () => { + const actual = jest.requireActual('./evaluate'); return { ...actual, postEvaluation: jest.fn((...args) => actual.postEvaluation(...args)), @@ -37,7 +38,7 @@ const statusResponse = { }; const http = { - fetch: jest.fn().mockResolvedValue(statusResponse), + post: jest.fn().mockResolvedValue(statusResponse), }; const toasts = { addError: jest.fn(), @@ -53,20 +54,23 @@ describe('usePerformEvaluation', () => { const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps)); await waitForNextUpdate(); - expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { - method: 'POST', + expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { body: '{"dataset":[],"evalPrompt":""}', headers: { 'Content-Type': 'application/json', }, query: { agents: undefined, + datasetName: undefined, evalModel: undefined, evaluationType: undefined, models: undefined, outputIndex: undefined, + projectName: undefined, + runName: undefined, }, signal: undefined, + version: API_VERSIONS.internal.v1, }); expect(toasts.addError).not.toHaveBeenCalled(); }); @@ -82,6 +86,8 @@ describe('usePerformEvaluation', () => { evaluationType: ['f', 'e'], models: ['h', 'g'], outputIndex: 'outputIndex', + projectName: 'test project', + runName: 'test run', }); return Promise.resolve(res); } catch (e) { @@ -92,20 +98,23 @@ describe('usePerformEvaluation', () => { const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps)); await waitForNextUpdate(); - expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { - method: 'POST', + expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { body: '{"dataset":["kewl"],"evalPrompt":"evalPrompt"}', headers: { 'Content-Type': 'application/json', }, query: { agents: 'c,d', + datasetName: undefined, evalModel: 'a,b', evaluationType: 'e,f', models: 'g,h', outputIndex: 'outputIndex', + projectName: 'test project', + runName: 'test run', }, signal: undefined, + version: API_VERSIONS.internal.v1, }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx similarity index 97% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx index 158f7159310ad5..30e95d9d804074 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx @@ -9,7 +9,7 @@ import { useMutation } from '@tanstack/react-query'; import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; import type { IToasts } from '@kbn/core-notifications-browser'; import { i18n } from '@kbn/i18n'; -import { postEvaluation } from '../../api'; +import { postEvaluation } from './evaluate'; const PERFORM_EVALUATION_MUTATION_KEY = ['elastic-assistant', 'perform-evaluation']; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx index f4fe4d7f8a407f..09cdf6717ca6c3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx @@ -27,20 +27,16 @@ import { import { css } from '@emotion/react'; import { FormattedMessage } from '@kbn/i18n-react'; +import type { GetEvaluateResponse, PostEvaluateResponse } from '@kbn/elastic-assistant-common'; import * as i18n from './translations'; import { useAssistantContext } from '../../../assistant_context'; import { useLoadConnectors } from '../../../connectorland/use_load_connectors'; import { getActionTypeTitle, getGenAiConfig } from '../../../connectorland/helpers'; import { PRECONFIGURED_CONNECTOR } from '../../../connectorland/translations'; -import { usePerformEvaluation } from './use_perform_evaluation'; +import { usePerformEvaluation } from '../../api/evaluate/use_perform_evaluation'; import { getApmLink, getDiscoverLink } from './utils'; -import { PostEvaluationResponse } from '../../api'; +import { useEvaluationData } from '../../api/evaluate/use_evaluation_data'; -/** - * See AGENT_EXECUTOR_MAP in `x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts` - * for the agent name -> executor mapping - */ -const DEFAULT_AGENTS = ['DefaultAgentExecutor', 'OpenAIFunctionsExecutor']; const DEFAULT_EVAL_TYPES_OPTIONS = [ { label: 'correctness' }, { label: 'esql-validator', disabled: true }, @@ -65,6 +61,11 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet } = usePerformEvaluation({ http, }); + const { data: evalData } = useEvaluationData({ http }); + const defaultAgents = useMemo( + () => (evalData as GetEvaluateResponse)?.agentExecutors ?? [], + [evalData] + ); // Run Details // Project Name @@ -195,8 +196,8 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet [selectedAgentOptions] ); const agentOptions = useMemo(() => { - return DEFAULT_AGENTS.map((label) => ({ label })); - }, []); + return defaultAgents.map((label) => ({ label })); + }, [defaultAgents]); // Evaluation // Evaluation Type @@ -283,12 +284,12 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet ]); const discoverLink = useMemo( - () => getDiscoverLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''), + () => getDiscoverLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''), [basePath, evalResponse] ); const apmLink = useMemo( - () => getApmLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''), + () => getApmLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''), [basePath, evalResponse] ); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 930374567533bf..3551ef6b126c3f 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -7,9 +7,9 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants'; import { - PostEvaluateBodyInputs, - PostEvaluatePathQueryInputs, -} from '../schemas/evaluate/post_evaluate'; + PostEvaluateRequestBodyInput, + PostEvaluateRequestQueryInput, +} from '@kbn/elastic-assistant-common'; export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -46,8 +46,8 @@ export const getPostEvaluateRequest = ({ body, query, }: { - body: PostEvaluateBodyInputs; - query: PostEvaluatePathQueryInputs; + body: PostEvaluateRequestBodyInput; + query: PostEvaluateRequestQueryInput; }) => requestMock.create({ body, diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts new file mode 100644 index 00000000000000..b36081ce4bead0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AgentExecutor } from './types'; +import { callAgentExecutor } from '../execute_custom_llm_chain'; +import { callOpenAIFunctionsExecutor } from './openai_functions_executor'; + +/** + * To support additional Agent Executors from the UI, add them to this map + * and reference your specific AgentExecutor function + */ +export const AGENT_EXECUTOR_MAP: Record = { + DefaultAgentExecutor: callAgentExecutor, + OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor, +}; diff --git a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts index 54040d3d1b58e6..291aa9d8c25193 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts @@ -12,8 +12,8 @@ import { chunk as createChunks } from 'lodash/fp'; import { Logger } from '@kbn/core/server'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer, RunCollectorCallbackHandler } from 'langchain/callbacks'; +import { Dataset } from '@kbn/elastic-assistant-common'; import { AgentExecutorEvaluatorWithMetadata } from '../langchain/executors/types'; -import { Dataset } from '../../schemas/evaluate/post_evaluate'; import { callAgentWithRetry, getMessageFromLangChainResponse } from './utils'; import { ResponseBody } from '../langchain/types'; import { isLangSmithEnabled, writeLangSmithFeedback } from '../../routes/evaluate/utils'; @@ -102,7 +102,6 @@ export const performEvaluation = async ({ const chunk = requestChunks.shift() ?? []; const chunkNumber = totalChunks - requestChunks.length; logger.info(`Prediction request chunk: ${chunkNumber} of ${totalChunks}`); - logger.debug(chunk); // Note, order is kept between chunk and dataset, and is preserved w/ Promise.allSettled const chunkResults = await Promise.allSettled(chunk.map((r) => r.request())); diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index bbc2c63381fc92..eec0a08ccb8cd3 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -43,6 +43,7 @@ import { GetRegisteredTools, } from './services/app_context'; import { getCapabilitiesRoute } from './routes/capabilities/get_capabilities_route'; +import { getEvaluateRoute } from './routes/evaluate/get_evaluate'; interface CreateRouteHandlerContextParams { core: CoreSetup; @@ -124,6 +125,7 @@ export class ElasticAssistantPlugin postActionsConnectorExecuteRoute(router, getElserId); // Evaluate postEvaluateRoute(router, getElserId); + getEvaluateRoute(router); // Capabilities getCapabilitiesRoute(router); return { diff --git a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts index 105e1676fb8089..7c470cdfc2d94b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts @@ -8,12 +8,17 @@ import { IKibanaResponse, IRouter } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import type { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; +import { + API_VERSIONS, + GetCapabilitiesResponse, + INTERNAL_API_ACCESS, +} from '@kbn/elastic-assistant-common'; import { CAPABILITIES } from '../../../common/constants'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { buildResponse } from '../../lib/build_response'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; +import { buildRouteValidationWithZod } from '../../schemas/common'; /** * Get the assistant capabilities for the requesting plugin @@ -23,7 +28,7 @@ import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; export const getCapabilitiesRoute = (router: IRouter) => { router.versioned .get({ - access: 'internal', + access: INTERNAL_API_ACCESS, path: CAPABILITIES, options: { tags: ['access:elasticAssistant'], @@ -31,8 +36,14 @@ export const getCapabilitiesRoute = (router: IRouter> => { const resp = buildResponse(response); diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts new file mode 100644 index 00000000000000..bc9922ef5f35a6 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type IKibanaResponse, IRouter } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + API_VERSIONS, + INTERNAL_API_ACCESS, + GetEvaluateResponse, +} from '@kbn/elastic-assistant-common'; +import { buildResponse } from '../../lib/build_response'; +import { ElasticAssistantRequestHandlerContext } from '../../types'; +import { EVALUATE } from '../../../common/constants'; +import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; +import { buildRouteValidationWithZod } from '../../schemas/common'; +import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; + +export const getEvaluateRoute = (router: IRouter) => { + router.versioned + .get({ + access: INTERNAL_API_ACCESS, + path: EVALUATE, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: { + response: { + 200: { + body: buildRouteValidationWithZod(GetEvaluateResponse), + }, + }, + }, + }, + async (context, request, response): Promise> => { + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + + // Validate evaluation feature is enabled + const pluginName = getPluginNameFromRequest({ + request, + defaultPluginName: DEFAULT_PLUGIN_NAME, + logger, + }); + const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); + if (!registeredFeatures.assistantModelEvaluation) { + return response.notFound(); + } + + try { + return response.ok({ body: { agentExecutors: Object.keys(AGENT_EXECUTOR_MAP) } }); + } catch (err) { + logger.error(err); + const error = transformError(err); + + const resp = buildResponse(response); + return resp.error({ + body: { error: error.message }, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts index 3ae64f1d89f3b3..64ec69fa5e943d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts @@ -9,17 +9,17 @@ import { postEvaluateRoute } from './post_evaluate'; import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getPostEvaluateRequest } from '../../__mocks__/request'; -import { - PostEvaluateBodyInputs, - PostEvaluatePathQueryInputs, -} from '../../schemas/evaluate/post_evaluate'; +import type { + PostEvaluateRequestBodyInput, + PostEvaluateRequestQueryInput, +} from '@kbn/elastic-assistant-common'; -const defaultBody: PostEvaluateBodyInputs = { +const defaultBody: PostEvaluateRequestBodyInput = { dataset: undefined, evalPrompt: undefined, }; -const defaultQueryParams: PostEvaluatePathQueryInputs = { +const defaultQueryParams: PostEvaluateRequestQueryInput = { agents: 'agents', datasetName: undefined, evaluationType: undefined, diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index aa041175b75ee0..33d19d6fb61e05 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -5,23 +5,23 @@ * 2.0. */ -import { IRouter, KibanaRequest } from '@kbn/core/server'; +import { type IKibanaResponse, IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { v4 as uuidv4 } from 'uuid'; +import { + API_VERSIONS, + INTERNAL_API_ACCESS, + PostEvaluateBody, + PostEvaluateRequestQuery, + PostEvaluateResponse, +} from '@kbn/elastic-assistant-common'; import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; -import { buildRouteValidation } from '../../schemas/common'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; import { EVALUATE } from '../../../common/constants'; -import { PostEvaluateBody, PostEvaluatePathQuery } from '../../schemas/evaluate/post_evaluate'; import { performEvaluation } from '../../lib/model_evaluator/evaluation'; -import { callAgentExecutor } from '../../lib/langchain/execute_custom_llm_chain'; -import { callOpenAIFunctionsExecutor } from '../../lib/langchain/executors/openai_functions_executor'; -import { - AgentExecutor, - AgentExecutorEvaluatorWithMetadata, -} from '../../lib/langchain/executors/types'; +import { AgentExecutorEvaluatorWithMetadata } from '../../lib/langchain/executors/types'; import { ActionsClientLlm } from '../../lib/langchain/llm/actions_client_llm'; import { indexEvaluations, @@ -30,15 +30,8 @@ import { import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils'; import { RequestBody } from '../../lib/langchain/types'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; - -/** - * To support additional Agent Executors from the UI, add them to this map - * and reference your specific AgentExecutor function - */ -const AGENT_EXECUTOR_MAP: Record = { - DefaultAgentExecutor: callAgentExecutor, - OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor, -}; +import { buildRouteValidationWithZod } from '../../schemas/common'; +import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; const DEFAULT_SIZE = 20; @@ -46,200 +39,215 @@ export const postEvaluateRoute = ( router: IRouter, getElser: GetElser ) => { - router.post( - { + router.versioned + .post({ + access: INTERNAL_API_ACCESS, path: EVALUATE, - validate: { - body: buildRouteValidation(PostEvaluateBody), - query: buildRouteValidation(PostEvaluatePathQuery), + options: { + tags: ['access:elasticAssistant'], }, - }, - async (context, request, response) => { - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; - - // Validate evaluation feature is enabled - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); - if (!registeredFeatures.assistantModelEvaluation) { - return response.notFound(); - } - - try { - const evaluationId = uuidv4(); - const { - evalModel, - evaluationType, - outputIndex, - datasetName, - projectName = 'default', - runName = evaluationId, - } = request.query; - const { dataset: customDataset = [], evalPrompt } = request.body; - const connectorIds = request.query.models?.split(',') || []; - const agentNames = request.query.agents?.split(',') || []; - - const dataset = - datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; - - logger.info('postEvaluateRoute:'); - logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); - logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); - logger.info(`Evaluation ID: ${evaluationId}`); - - const totalExecutions = connectorIds.length * agentNames.length * dataset.length; - logger.info('Creating agents:'); - logger.info(`\tconnectors/models: ${connectorIds.length}`); - logger.info(`\tagents: ${agentNames.length}`); - logger.info(`\tdataset: ${dataset.length}`); - logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); - if (totalExecutions > 50) { - logger.warn( - `Total baseline agent executions >= 50! This may take a while, and cost some money...` - ); + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: { + request: { + body: buildRouteValidationWithZod(PostEvaluateBody), + query: buildRouteValidationWithZod(PostEvaluateRequestQuery), + }, + response: { + 200: { + body: buildRouteValidationWithZod(PostEvaluateResponse), + }, + }, + }, + }, + async (context, request, response): Promise> => { + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; + + // Validate evaluation feature is enabled + const pluginName = getPluginNameFromRequest({ + request, + defaultPluginName: DEFAULT_PLUGIN_NAME, + logger, + }); + const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); + if (!registeredFeatures.assistantModelEvaluation) { + return response.notFound(); } - // Get the actions plugin start contract from the request context for the agents - const actions = (await context.elasticAssistant).actions; + try { + const evaluationId = uuidv4(); + const { + evalModel, + evaluationType, + outputIndex, + datasetName, + projectName = 'default', + runName = evaluationId, + } = request.query; + const { dataset: customDataset = [], evalPrompt } = request.body; + const connectorIds = request.query.models?.split(',') || []; + const agentNames = request.query.agents?.split(',') || []; + + const dataset = + datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; + + logger.info('postEvaluateRoute:'); + logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); + logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); + logger.info(`Evaluation ID: ${evaluationId}`); + + const totalExecutions = connectorIds.length * agentNames.length * dataset.length; + logger.info('Creating agents:'); + logger.info(`\tconnectors/models: ${connectorIds.length}`); + logger.info(`\tagents: ${agentNames.length}`); + logger.info(`\tdataset: ${dataset.length}`); + logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); + if (totalExecutions > 50) { + logger.warn( + `Total baseline agent executions >= 50! This may take a while, and cost some money...` + ); + } + + // Get the actions plugin start contract from the request context for the agents + const actions = (await context.elasticAssistant).actions; + + // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm + const actionsClient = await actions.getActionsClientWithRequest(request); + const connectors = await actionsClient.getBulk({ + ids: connectorIds, + throwIfSystemAction: false, + }); - // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm - const actionsClient = await actions.getActionsClientWithRequest(request); - const connectors = await actionsClient.getBulk({ - ids: connectorIds, - throwIfSystemAction: false, - }); + // Fetch any tools registered by the request's originating plugin + const assistantTools = (await context.elasticAssistant).getRegisteredTools( + 'securitySolution' + ); - // Fetch any tools registered by the request's originating plugin - const assistantTools = (await context.elasticAssistant).getRegisteredTools( - 'securitySolution' - ); - - // Get a scoped esClient for passing to the agents for retrieval, and - // writing results to the output index - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - - // Default ELSER model - const elserId = await getElser(request, (await context.core).savedObjects.getClient()); - - // Skeleton request from route to pass to the agents - // params will be passed to the actions executor - const skeletonRequest: KibanaRequest = { - ...request, - body: { - alertsIndexPattern: '', - allow: [], - allowReplacement: [], - params: { - subAction: 'invokeAI', - subActionParams: { - messages: [], + // Get a scoped esClient for passing to the agents for retrieval, and + // writing results to the output index + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + // Default ELSER model + const elserId = await getElser(request, (await context.core).savedObjects.getClient()); + + // Skeleton request from route to pass to the agents + // params will be passed to the actions executor + const skeletonRequest: KibanaRequest = { + ...request, + body: { + alertsIndexPattern: '', + allow: [], + allowReplacement: [], + params: { + subAction: 'invokeAI', + subActionParams: { + messages: [], + }, }, + replacements: {}, + size: DEFAULT_SIZE, + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: true, }, - replacements: {}, - size: DEFAULT_SIZE, - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: true, - }, - }; - - // Create an array of executor functions to call in batches - // One for each connector/model + agent combination - // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator - const agents: AgentExecutorEvaluatorWithMetadata[] = []; - connectorIds.forEach((connectorId) => { - agentNames.forEach((agentName) => { - logger.info(`Creating agent: ${connectorId} + ${agentName}`); - const llmType = getLlmType(connectorId, connectors); - const connectorName = - getConnectorName(connectorId, connectors) ?? '[unknown connector]'; - const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; - agents.push({ - agentEvaluator: (langChainMessages, exampleId) => - AGENT_EXECUTOR_MAP[agentName]({ - actions, - isEnabledKnowledgeBase: true, - assistantTools, - connectorId, - esClient, - elserId, - langChainMessages, - llmType, - logger, - request: skeletonRequest, - kbResource: ESQL_RESOURCE, - telemetry, - traceOptions: { - exampleId, - projectName, - runName: detailedRunName, - evaluationId, - tags: [ - 'security-assistant-prediction', - ...(connectorName != null ? [connectorName] : []), - runName, - ], - tracers: getLangSmithTracer(detailedRunName, exampleId, logger), - }, - }), - metadata: { - connectorName, - runName: detailedRunName, - }, + }; + + // Create an array of executor functions to call in batches + // One for each connector/model + agent combination + // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator + const agents: AgentExecutorEvaluatorWithMetadata[] = []; + connectorIds.forEach((connectorId) => { + agentNames.forEach((agentName) => { + logger.info(`Creating agent: ${connectorId} + ${agentName}`); + const llmType = getLlmType(connectorId, connectors); + const connectorName = + getConnectorName(connectorId, connectors) ?? '[unknown connector]'; + const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; + agents.push({ + agentEvaluator: (langChainMessages, exampleId) => + AGENT_EXECUTOR_MAP[agentName]({ + actions, + isEnabledKnowledgeBase: true, + assistantTools, + connectorId, + esClient, + elserId, + langChainMessages, + llmType, + logger, + request: skeletonRequest, + kbResource: ESQL_RESOURCE, + telemetry, + traceOptions: { + exampleId, + projectName, + runName: detailedRunName, + evaluationId, + tags: [ + 'security-assistant-prediction', + ...(connectorName != null ? [connectorName] : []), + runName, + ], + tracers: getLangSmithTracer(detailedRunName, exampleId, logger), + }, + }), + metadata: { + connectorName, + runName: detailedRunName, + }, + }); }); }); - }); - logger.info(`Agents created: ${agents.length}`); - - // Evaluator Model is optional to support just running predictions - const evaluatorModel = - evalModel == null || evalModel === '' - ? undefined - : new ActionsClientLlm({ - actions, - connectorId: evalModel, - request: skeletonRequest, - logger, - }); + logger.info(`Agents created: ${agents.length}`); - const { evaluationResults, evaluationSummary } = await performEvaluation({ - agentExecutorEvaluators: agents, - dataset, - evaluationId, - evaluatorModel, - evaluationPrompt: evalPrompt, - evaluationType, - logger, - runName, - }); + // Evaluator Model is optional to support just running predictions + const evaluatorModel = + evalModel == null || evalModel === '' + ? undefined + : new ActionsClientLlm({ + actions, + connectorId: evalModel, + request: skeletonRequest, + logger, + }); + + const { evaluationResults, evaluationSummary } = await performEvaluation({ + agentExecutorEvaluators: agents, + dataset, + evaluationId, + evaluatorModel, + evaluationPrompt: evalPrompt, + evaluationType, + logger, + runName, + }); - logger.info(`Writing evaluation results to index: ${outputIndex}`); - await setupEvaluationIndex({ esClient, index: outputIndex, logger }); - await indexEvaluations({ - esClient, - evaluationResults, - evaluationSummary, - index: outputIndex, - logger, - }); + logger.info(`Writing evaluation results to index: ${outputIndex}`); + await setupEvaluationIndex({ esClient, index: outputIndex, logger }); + await indexEvaluations({ + esClient, + evaluationResults, + evaluationSummary, + index: outputIndex, + logger, + }); - return response.ok({ - body: { evaluationId, success: true }, - }); - } catch (err) { - logger.error(err); - const error = transformError(err); - - const resp = buildResponse(response); - return resp.error({ - body: { success: false, error: error.message }, - statusCode: error.statusCode, - }); + return response.ok({ + body: { evaluationId, success: true }, + }); + } catch (err) { + logger.error(err); + const error = transformError(err); + + const resp = buildResponse(response); + return resp.error({ + body: { success: false, error: error.message }, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts index 550e89667256e3..11f8cb9c2f692b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts @@ -12,7 +12,7 @@ import type { Logger } from '@kbn/core/server'; import type { Run } from 'langsmith/schemas'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer } from 'langchain/callbacks'; -import { Dataset } from '../../schemas/evaluate/post_evaluate'; +import { Dataset } from '@kbn/elastic-assistant-common'; /** * Returns the LangChain `llmType` for the given connectorId/connectors diff --git a/x-pack/plugins/elastic_assistant/server/schemas/common.ts b/x-pack/plugins/elastic_assistant/server/schemas/common.ts index 00e97a9326c5e6..5e847aef69fc08 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/common.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/common.ts @@ -14,6 +14,8 @@ import type { RouteValidationResultFactory, RouteValidationError, } from '@kbn/core/server'; +import type { TypeOf, ZodType } from 'zod'; +import { stringifyZodError } from '@kbn/zod-helpers'; type RequestValidationResult = | { @@ -36,3 +38,14 @@ export const buildRouteValidation = (validatedInput: A) => validationResult.ok(validatedInput) ) ); + +export const buildRouteValidationWithZod = + >(schema: T): RouteValidationFunction => + (inputValue: unknown, validationResult: RouteValidationResultFactory) => { + const decoded = schema.safeParse(inputValue); + if (decoded.success) { + return validationResult.ok(decoded.data); + } else { + return validationResult.badRequest(stringifyZodError(decoded.error)); + } + }; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts deleted file mode 100644 index f520bf9bf93b6b..00000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -/** Validates Output Index starts with `.kibana-elastic-ai-assistant-` */ -const outputIndex = new t.Type( - 'OutputIndexPrefixed', - (input): input is string => - typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-'), - (input, context) => - typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-') - ? t.success(input) - : t.failure( - input, - context, - `Type error: Output Index does not start with '.kibana-elastic-ai-assistant-'` - ), - t.identity -); - -/** Validates the URL path of a POST request to the `/evaluate` endpoint */ -export const PostEvaluatePathQuery = t.type({ - agents: t.string, - datasetName: t.union([t.string, t.undefined]), - evaluationType: t.union([t.string, t.undefined]), - evalModel: t.union([t.string, t.undefined]), - models: t.string, - outputIndex, - projectName: t.union([t.string, t.undefined]), - runName: t.union([t.string, t.undefined]), -}); - -export type PostEvaluatePathQueryInputs = t.TypeOf; - -export type DatasetItem = t.TypeOf; -export const DatasetItem = t.type({ - id: t.union([t.string, t.undefined]), - input: t.string, - reference: t.string, - tags: t.union([t.array(t.string), t.undefined]), - prediction: t.union([t.string, t.undefined]), -}); - -export type Dataset = t.TypeOf; -export const Dataset = t.array(DatasetItem); - -/** Validates the body of a POST request to the `/evaluate` endpoint */ -export const PostEvaluateBody = t.type({ - dataset: t.union([Dataset, t.undefined]), - evalPrompt: t.union([t.string, t.undefined]), -}); - -export type PostEvaluateBodyInputs = t.TypeOf; diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index dfca7893b20365..2717da8d33a3ac 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -35,6 +35,7 @@ "@kbn/core-analytics-server", "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", + "@kbn/zod-helpers", ], "exclude": [ "target/**/*",