From 31026a0b65c8d6bf6f7667ab47b856b9d3b2af9c Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Wed, 31 Jan 2024 17:52:03 +0100 Subject: [PATCH 01/20] [ES|QL] Add quick fixes feature to query validation errors (#175553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR improves the UX of a query editing experience via the quick fix feature, typical of a full IDE. This is a bit of an experiment and a small enhancement already, but I'm curious on more use cases for this quick fixes option. I've addressed some simple ones here, but let me know if you think there are scenarios you would like to be covered. Spellchecks fixes are computed using a levenshtein distance of 3, same as on the ES counterpart. Current supported features: * index spellcheck quick fix ![fix_indexes](https://github.com/elastic/kibana/assets/924948/dfd74f3d-28c2-4880-b771-42ea42e418d2) * wildcard support ![fix_indexes_wildcard](https://github.com/elastic/kibana/assets/924948/058c1e03-f4ae-40d2-ac73-e6da2b1ccde3) * policy spellcheck quick fix ![fix_policies](https://github.com/elastic/kibana/assets/924948/24340f0a-d349-4db2-b009-97193025cf66) * field/column spellcheck quick fix ![fix_fields](https://github.com/elastic/kibana/assets/924948/dda7e1bb-13bc-410f-a578-ef0feaed0a0f) * function spellcheck quick fix ![fix_functions](https://github.com/elastic/kibana/assets/924948/40fa240a-034d-4ced-a23d-b970c9ebdc27) * wrong quotes quick fix for literal strings ![fix_wrong_quotes](https://github.com/elastic/kibana/assets/924948/5f471b53-39a7-4b07-be2a-d0da27a17315) * unquoted field/column quick fix ![fix_unquoted](https://github.com/elastic/kibana/assets/924948/c2ed7b58-a10e-4fd2-b51d-484b3b2a09e7) Testing this feature I've also found a subtle bug in the autocomplete code on unknown functions 😅 . This feature requires some monaco additional sub-dependencies, so bundle size can increase. I was looking for a `Fix all` action as well, but I suspect there's another route for that as error markers provided to the codeAction provider are scoped by the hovered area in the editor, so multiple fixes are only possible for the same hovered area and in this case I'm already handling that manually for specific cases (i.e. unquoted field triggers 2 syntax + 1 validation errors). --------- Co-authored-by: Eyo Okon Eyo Co-authored-by: Aleh Zasypkin Co-authored-by: Stratoula Kalafateli --- packages/kbn-monaco/BUILD.bazel | 1 + packages/kbn-monaco/src/esql/language.ts | 21 + .../lib/ast/autocomplete/autocomplete.test.ts | 4 +- .../esql/lib/ast/autocomplete/autocomplete.ts | 16 +- .../esql/lib/ast/autocomplete/factories.ts | 9 +- .../esql/lib/ast/code_actions/index.test.ts | 257 ++++++++++++ .../src/esql/lib/ast/code_actions/index.ts | 383 ++++++++++++++++++ .../src/esql/lib/ast/definitions/aggs.ts | 4 + .../src/esql/lib/ast/definitions/builtin.ts | 90 ++-- .../src/esql/lib/ast/definitions/commands.ts | 4 + .../src/esql/lib/ast/definitions/functions.ts | 1 + .../src/esql/lib/ast/definitions/options.ts | 1 + .../src/esql/lib/ast/definitions/types.ts | 6 +- .../src/esql/lib/ast/shared/context.ts | 2 +- .../src/esql/lib/ast/shared/helpers.ts | 18 + .../src/esql/lib/ast/shared/monaco_utils.ts | 54 +++ .../esql/lib/ast/shared/resources_helpers.ts | 6 + packages/kbn-monaco/src/esql/lib/ast/types.ts | 1 + .../src/esql/lib/ast/validation/errors.ts | 10 +- .../src/esql/lib/ast/validation/validation.ts | 10 +- .../src/esql/lib/monaco/esql_ast_provider.ts | 56 +-- packages/kbn-monaco/src/monaco_imports.ts | 7 + packages/kbn-monaco/src/types.ts | 2 + .../src/text_based_languages_editor.tsx | 11 + .../code_editor/impl/code_editor.tsx | 13 + 25 files changed, 879 insertions(+), 108 deletions(-) create mode 100644 packages/kbn-monaco/src/esql/lib/ast/code_actions/index.test.ts create mode 100644 packages/kbn-monaco/src/esql/lib/ast/code_actions/index.ts create mode 100644 packages/kbn-monaco/src/esql/lib/ast/shared/monaco_utils.ts diff --git a/packages/kbn-monaco/BUILD.bazel b/packages/kbn-monaco/BUILD.bazel index 232ebdb2ba39bc..95833afd585b8b 100644 --- a/packages/kbn-monaco/BUILD.bazel +++ b/packages/kbn-monaco/BUILD.bazel @@ -29,6 +29,7 @@ SHARED_DEPS = [ "@npm//antlr4ts", "@npm//monaco-editor", "@npm//monaco-yaml", + "@npm//js-levenshtein", ] webpack_cli( diff --git a/packages/kbn-monaco/src/esql/language.ts b/packages/kbn-monaco/src/esql/language.ts index 33a911b2272d66..c37e19fb81d93d 100644 --- a/packages/kbn-monaco/src/esql/language.ts +++ b/packages/kbn-monaco/src/esql/language.ts @@ -111,4 +111,25 @@ export const ESQLLang: CustomLangModuleType = { }, }; }, + + getCodeActionProvider: (callbacks?: ESQLCallbacks): monaco.languages.CodeActionProvider => { + return { + async provideCodeActions( + model /** ITextModel*/, + range /** Range*/, + context /** CodeActionContext*/, + token /** CancellationToken*/ + ) { + const astAdapter = new ESQLAstAdapter( + (...uris) => workerProxyService.getWorker(uris), + callbacks + ); + const actions = await astAdapter.codeAction(model, range, context); + return { + actions, + dispose: () => {}, + }; + }, + }; + }, }; diff --git a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts index 12a1db12392d5f..c3597ba00c4e22 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts @@ -121,8 +121,8 @@ function getFunctionSignaturesByReturnType( } return true; }) - .map(({ builtin: isBuiltinFn, name, signatures, ...defRest }) => - isBuiltinFn ? `${name} $0` : `${name}($0)` + .map(({ type, name, signatures, ...defRest }) => + type === 'builtin' ? `${name} $0` : `${name}($0)` ); } diff --git a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts index 80fdb22ee207a1..4cf468febbfd00 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts @@ -37,7 +37,6 @@ import { import { collectVariables, excludeVariablesFromCurrentCommand } from '../shared/variables'; import type { AstProviderFn, - ESQLAst, ESQLAstItem, ESQLCommand, ESQLCommandMode, @@ -72,6 +71,7 @@ import { import { EDITOR_MARKER } from '../shared/constants'; import { getAstContext, removeMarkerArgFromArgsList } from '../shared/context'; import { + buildQueryUntilPreviousCommand, getFieldsByTypeHelper, getPolicyHelper, getSourcesHelper, @@ -187,7 +187,7 @@ export async function suggest( const astContext = getAstContext(innerText, ast, offset); // build the correct query to fetch the list of fields - const queryForFields = buildQueryForFields(ast, finalText); + const queryForFields = buildQueryUntilPreviousCommand(ast, finalText); const { getFieldsByType, getFieldsMap } = getFieldsByTypeRetriever( queryForFields, resourceRetriever @@ -260,11 +260,6 @@ export async function suggest( return []; } -export function buildQueryForFields(ast: ESQLAst, queryString: string) { - const prevCommand = ast[Math.max(ast.length - 2, 0)]; - return prevCommand ? queryString.substring(0, prevCommand.location.max + 1) : queryString; -} - function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) { const helpers = getFieldsByTypeHelper(queryString, resourceRetriever); return { @@ -812,7 +807,7 @@ async function getBuiltinFunctionNextArgument( // technically another boolean value should be suggested, but it is a better experience // to actually suggest a wider set of fields/functions [ - finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.builtin + finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.type === 'builtin' ? 'any' : finalType, ], @@ -1013,7 +1008,7 @@ async function getFunctionArgsSuggestions( ? { ...suggestion, insertText: - hasMoreMandatoryArgs && !fnDefinition.builtin + hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' ? `${suggestion.insertText},` : suggestion.insertText, } @@ -1023,7 +1018,8 @@ async function getFunctionArgsSuggestions( return suggestions.map(({ insertText, ...rest }) => ({ ...rest, - insertText: hasMoreMandatoryArgs && !fnDefinition.builtin ? `${insertText},` : insertText, + insertText: + hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' ? `${insertText},` : insertText, })); } diff --git a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts index 9fc026852f999e..45837ed42a86fd 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts @@ -18,7 +18,7 @@ import { CommandOptionsDefinition, CommandModeDefinition, } from '../definitions/types'; -import { getCommandDefinition } from '../shared/helpers'; +import { getCommandDefinition, shouldBeQuotedText } from '../shared/helpers'; import { buildDocumentation, buildFunctionDocumentation } from './documentation_util'; const allFunctions = statsAggregationFunctionDefinitions.concat(evalFunctionsDefinitions); @@ -28,11 +28,8 @@ export const TRIGGER_SUGGESTION_COMMAND = { id: 'editor.action.triggerSuggest', }; -function getSafeInsertText(text: string, { dashSupported }: { dashSupported?: boolean } = {}) { - if (dashSupported) { - return /[^a-zA-Z\d_\.@-]/.test(text) ? `\`${text}\`` : text; - } - return /[^a-zA-Z\d_\.@]/.test(text) ? `\`${text}\`` : text; +function getSafeInsertText(text: string, options: { dashSupported?: boolean } = {}) { + return shouldBeQuotedText(text, options) ? `\`${text}\`` : text; } export function getAutocompleteFunctionDefinition(fn: FunctionDefinition) { diff --git a/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.test.ts b/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.test.ts new file mode 100644 index 00000000000000..58d825ef68aea1 --- /dev/null +++ b/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.test.ts @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EditorError } from '../../../../types'; +import { CharStreams } from 'antlr4ts'; +import { getActions } from '.'; +import { getParser, ROOT_STATEMENT } from '../../antlr_facade'; +import { ESQLErrorListener } from '../../monaco/esql_error_listener'; +import { AstListener } from '../ast_factory'; +import { wrapAsMonacoMessage } from '../shared/monaco_utils'; +import { ESQLAst } from '../types'; +import { validateAst } from '../validation/validation'; +import { monaco } from '../../../../monaco_imports'; +import { getAllFunctions } from '../shared/helpers'; + +function getCallbackMocks() { + return { + getFieldsFor: jest.fn(async ({ query }) => + /enrich/.test(query) + ? [ + { name: 'otherField', type: 'string' }, + { name: 'yetAnotherField', type: 'number' }, + ] + : /unsupported_index/.test(query) + ? [{ name: 'unsupported_field', type: 'unsupported' }] + : [ + ...['string', 'number', 'date', 'boolean', 'ip'].map((type) => ({ + name: `${type}Field`, + type, + })), + { name: 'geoPointField', type: 'geo_point' }, + { name: 'any#Char$Field', type: 'number' }, + { name: 'kubernetes.something.something', type: 'number' }, + { + name: `listField`, + type: `list`, + }, + { name: '@timestamp', type: 'date' }, + ] + ), + getSources: jest.fn(async () => + ['index', '.secretIndex', 'my-index'].map((name) => ({ + name, + hidden: name.startsWith('.'), + })) + ), + getPolicies: jest.fn(async () => [ + { + name: 'policy', + sourceIndices: ['enrichIndex1'], + matchField: 'otherStringField', + enrichFields: ['other-field', 'yetAnotherField'], + }, + { + name: 'policy[]', + sourceIndices: ['enrichIndex1'], + matchField: 'otherStringField', + enrichFields: ['other-field', 'yetAnotherField'], + }, + ]), + }; +} + +const getAstAndErrors = async ( + text: string | undefined +): Promise<{ + errors: EditorError[]; + ast: ESQLAst; +}> => { + if (text == null) { + return { ast: [], errors: [] }; + } + const errorListener = new ESQLErrorListener(); + const parseListener = new AstListener(); + const parser = getParser(CharStreams.fromString(text), errorListener, parseListener); + + parser[ROOT_STATEMENT](); + + return { ...parseListener.getAst(), errors: errorListener.getErrors() }; +}; + +function createModelAndRange(text: string) { + return { + model: { getValue: () => text } as monaco.editor.ITextModel, + range: {} as monaco.Range, + }; +} + +function createMonacoContext(errors: EditorError[]): monaco.languages.CodeActionContext { + return { + markers: errors, + trigger: 1, + }; +} + +/** + * There are different wats to test the code here: one is a direct unit test of the feature, another is + * an integration test passing from the query statement validation. The latter is more realistic, but + * a little bit more tricky to setup. This function will encapsulate all the complexity + */ +function testQuickFixesFn( + statement: string, + expectedFixes: string[] = [], + options: { equalityCheck?: 'include' | 'equal' } = {}, + { only, skip }: { only?: boolean; skip?: boolean } = {} +) { + const testFn = only ? it.only : skip ? it.skip : it; + const { model, range } = createModelAndRange(statement); + testFn(`${statement} => ["${expectedFixes.join('","')}"]`, async () => { + const callbackMocks = getCallbackMocks(); + const { errors } = await validateAst(statement, getAstAndErrors, callbackMocks); + + const monacoErrors = wrapAsMonacoMessage('error', statement, errors); + const context = createMonacoContext(monacoErrors); + const actions = await getActions(model, range, context, getAstAndErrors, callbackMocks); + const edits = actions.map( + ({ edit }) => (edit?.edits[0] as monaco.languages.IWorkspaceTextEdit).textEdit.text + ); + expect(edits).toEqual( + !options || !options.equalityCheck || options.equalityCheck === 'equal' + ? expectedFixes + : expect.arrayContaining(expectedFixes) + ); + }); +} + +type TestArgs = [string, string[], { equalityCheck?: 'include' | 'equal' }]; + +// Make only and skip work with our custom wrapper +const testQuickFixes = Object.assign(testQuickFixesFn, { + skip: (...args: TestArgs) => { + const paddingArgs = ['equal'].slice(args.length - 2); + return testQuickFixesFn(...((args.length > 1 ? [...args, ...paddingArgs] : args) as TestArgs), { + skip: true, + }); + }, + only: (...args: TestArgs) => { + const paddingArgs = ['equal'].slice(args.length - 2); + return testQuickFixesFn(...((args.length > 1 ? [...args, ...paddingArgs] : args) as TestArgs), { + only: true, + }); + }, +}); + +describe('quick fixes logic', () => { + describe('fixing index spellchecks', () => { + // No error, no quick action + testQuickFixes('FROM index', []); + testQuickFixes('FROM index2', ['index']); + testQuickFixes('FROM myindex', ['index', 'my-index']); + // wildcards + testQuickFixes('FROM index*', []); + testQuickFixes('FROM ind*', []); + testQuickFixes('FROM end*', ['ind*']); + testQuickFixes('FROM endex*', ['index']); + // Too far for the levenstein distance and should not fix with a hidden index + testQuickFixes('FROM secretIndex', []); + testQuickFixes('FROM secretIndex2', []); + }); + + describe('fixing fields spellchecks', () => { + for (const command of ['KEEP', 'DROP', 'EVAL']) { + testQuickFixes(`FROM index | ${command} stringField`, []); + // strongField => stringField + testQuickFixes(`FROM index | ${command} strongField`, ['stringField']); + testQuickFixes(`FROM index | ${command} numberField, strongField`, ['stringField']); + } + testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField']); + testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField']); + testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField']); + testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField']); + testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField']); + // This levarage the knowledge of the enrich policy fields to suggest the right field + testQuickFixes(`FROM index | ENRICH policy | KEEP yetAnotherField2`, ['yetAnotherField']); + testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField']); + testQuickFixes(`FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, [ + 'yetAnotherField', + ]); + }); + + describe('fixing policies spellchecks', () => { + testQuickFixes(`FROM index | ENRICH poli`, ['policy']); + testQuickFixes(`FROM index | ENRICH mypolicy`, ['policy']); + testQuickFixes(`FROM index | ENRICH policy[`, ['policy', 'policy[]']); + }); + + describe('fixing function spellchecks', () => { + function toFunctionSignature(name: string) { + return `${name}()`; + } + // it should be strange enough to make the function invalid + const BROKEN_PREFIX = 'Q'; + for (const fn of getAllFunctions({ type: 'eval' })) { + // add an A to the function name to make it invalid + testQuickFixes( + `FROM index | EVAL ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + testQuickFixes( + `FROM index | EVAL var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + testQuickFixes( + `FROM index | STATS avg(${BROKEN_PREFIX}${fn.name}())`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + testQuickFixes( + `FROM index | STATS avg(numberField) BY ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + testQuickFixes( + `FROM index | STATS avg(numberField) BY var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + } + for (const fn of getAllFunctions({ type: 'agg' })) { + // add an A to the function name to make it invalid + testQuickFixes( + `FROM index | STATS ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + testQuickFixes( + `FROM index | STATS var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include' } + ); + } + // it should preserve the arguments + testQuickFixes(`FROM index | EVAL rAund(numberField)`, ['round(numberField)'], { + equalityCheck: 'include', + }); + testQuickFixes(`FROM index | STATS AVVG(numberField)`, ['avg(numberField)'], { + equalityCheck: 'include', + }); + }); + + describe('fixing wrong quotes', () => { + testQuickFixes(`FROM index | WHERE stringField like 'asda'`, ['"asda"']); + testQuickFixes(`FROM index | WHERE stringField not like 'asda'`, ['"asda"']); + }); + + describe('fixing unquoted field names', () => { + testQuickFixes('FROM index | DROP any#Char$Field', ['`any#Char$Field`']); + testQuickFixes('FROM index | DROP numberField, any#Char$Field', ['`any#Char$Field`']); + }); +}); diff --git a/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.ts b/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.ts new file mode 100644 index 00000000000000..94079d1693b7f4 --- /dev/null +++ b/packages/kbn-monaco/src/esql/lib/ast/code_actions/index.ts @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { i18n } from '@kbn/i18n'; +import levenshtein from 'js-levenshtein'; +import type { monaco } from '../../../../monaco_imports'; +import { + getFieldsByTypeHelper, + getPolicyHelper, + getSourcesHelper, +} from '../shared/resources_helpers'; +import { getAllFunctions, isSourceItem, shouldBeQuotedText } from '../shared/helpers'; +import { ESQLCallbacks } from '../shared/types'; +import { AstProviderFn, ESQLAst } from '../types'; +import { buildQueryForFieldsFromSource } from '../validation/helpers'; + +type GetSourceFn = () => Promise; +type GetFieldsByTypeFn = (type: string | string[], ignored?: string[]) => Promise; +type GetPoliciesFn = () => Promise; +type GetPolicyFieldsFn = (name: string) => Promise; + +interface Callbacks { + getSources: GetSourceFn; + getFieldsByType: GetFieldsByTypeFn; + getPolicies: GetPoliciesFn; + getPolicyFields: GetPolicyFieldsFn; +} + +function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) { + const helpers = getFieldsByTypeHelper(queryString, resourceRetriever); + return { + getFieldsByType: async (expectedType: string | string[] = 'any', ignored: string[] = []) => { + const fields = await helpers.getFieldsByType(expectedType, ignored); + return fields; + }, + getFieldsMap: helpers.getFieldsMap, + }; +} + +function getPolicyRetriever(resourceRetriever?: ESQLCallbacks) { + const helpers = getPolicyHelper(resourceRetriever); + return { + getPolicies: async () => { + const policies = await helpers.getPolicies(); + return policies.map(({ name }) => name); + }, + getPolicyFields: async (policy: string) => { + const metadata = await helpers.getPolicyMetadata(policy); + return metadata?.enrichFields || []; + }, + }; +} + +function getSourcesRetriever(resourceRetriever?: ESQLCallbacks) { + const helper = getSourcesHelper(resourceRetriever); + return async () => { + const list = (await helper()) || []; + // hide indexes that start with . + return list.filter(({ hidden }) => !hidden).map(({ name }) => name); + }; +} + +export const getCompatibleFunctionDefinitions = ( + command: string, + option: string | undefined, + returnTypes?: string[], + ignored: string[] = [] +): string[] => { + const fnSupportedByCommand = getAllFunctions({ type: ['eval', 'agg'] }).filter( + ({ name, supportedCommands, supportedOptions }) => + (option ? supportedOptions?.includes(option) : supportedCommands.includes(command)) && + !ignored.includes(name) + ); + if (!returnTypes) { + return fnSupportedByCommand.map(({ name }) => name); + } + return fnSupportedByCommand + .filter((mathDefinition) => + mathDefinition.signatures.some( + (signature) => returnTypes[0] === 'any' || returnTypes.includes(signature.returnType) + ) + ) + .map(({ name }) => name); +}; + +function createAction( + title: string, + solution: string, + error: monaco.editor.IMarkerData, + uri: monaco.Uri +) { + return { + title, + diagnostics: [error], + kind: 'quickfix', + edit: { + edits: [ + { + resource: uri, + textEdit: { + range: error, + text: solution, + }, + versionId: undefined, + }, + ], + }, + isPreferred: true, + }; +} + +async function getSpellingPossibilities(fn: () => Promise, errorText: string) { + const allPossibilities = await fn(); + const allSolutions = allPossibilities.reduce((solutions, item) => { + const distance = levenshtein(item, errorText); + if (distance < 3) { + solutions.push(item); + } + return solutions; + }, [] as string[]); + // filter duplicates + return Array.from(new Set(allSolutions)); +} + +async function getSpellingActionForColumns( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + queryString: string, + ast: ESQLAst, + { getFieldsByType, getPolicies, getPolicyFields }: Callbacks +) { + const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); + // @TODO add variables support + const possibleFields = await getSpellingPossibilities(async () => { + const availableFields = await getFieldsByType('any'); + const enrichPolicies = ast.filter(({ name }) => name === 'enrich'); + if (enrichPolicies.length) { + const enrichPolicyNames = enrichPolicies.flatMap(({ args }) => + args.filter(isSourceItem).map(({ name }) => name) + ); + const enrichFields = await Promise.all(enrichPolicyNames.map(getPolicyFields)); + availableFields.push(...enrichFields.flat()); + } + return availableFields; + }, errorText); + return wrapIntoSpellingChangeAction(error, uri, possibleFields); +} + +async function getQuotableActionForColumns( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + queryString: string, + ast: ESQLAst, + { getFieldsByType }: Callbacks +) { + const commandEndIndex = ast.find((command) => command.location.max > error.endColumn)?.location + .max; + // the error received is unknwonColumn here, but look around the column to see if there's more + // which broke the grammar and the validation code couldn't identify as unquoted column + const remainingCommandText = queryString.substring( + error.endColumn - 1, + commandEndIndex ? commandEndIndex + 1 : undefined + ); + const stopIndex = Math.max( + /,/.test(remainingCommandText) + ? remainingCommandText.indexOf(',') + : /\s/.test(remainingCommandText) + ? remainingCommandText.indexOf(' ') + : remainingCommandText.length, + 0 + ); + const possibleUnquotedText = queryString.substring( + error.endColumn - 1, + error.endColumn + stopIndex + ); + const errorText = queryString + .substring(error.startColumn - 1, error.endColumn + possibleUnquotedText.length) + .trimEnd(); + const actions = []; + if (shouldBeQuotedText(errorText)) { + const availableFields = new Set(await getFieldsByType('any')); + const solution = `\`${errorText}\``; + if (availableFields.has(errorText) || availableFields.has(solution)) { + actions.push( + createAction( + i18n.translate('monaco.esql.quickfix.replaceWithSolution', { + defaultMessage: 'Did you mean {solution} ?', + values: { + solution, + }, + }), + solution, + { ...error, endColumn: error.startColumn + errorText.length }, // override the location + uri + ) + ); + } + } + return actions; +} + +async function getSpellingActionForIndex( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + queryString: string, + ast: ESQLAst, + { getSources }: Callbacks +) { + const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); + const possibleSources = await getSpellingPossibilities(async () => { + // Handle fuzzy names via truncation to test levenstein distance + const sources = await getSources(); + if (errorText.endsWith('*')) { + return sources.map((source) => + source.length > errorText.length ? source.substring(0, errorText.length - 1) + '*' : source + ); + } + return sources; + }, errorText); + return wrapIntoSpellingChangeAction(error, uri, possibleSources); +} + +async function getSpellingActionForPolicies( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + queryString: string, + ast: ESQLAst, + { getPolicies }: Callbacks +) { + const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); + const possiblePolicies = await getSpellingPossibilities(getPolicies, errorText); + return wrapIntoSpellingChangeAction(error, uri, possiblePolicies); +} + +async function getSpellingActionForFunctions( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + queryString: string, + ast: ESQLAst +) { + const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); + // fallback to the last command if not found + const commandContext = + ast.find((command) => command.location.max > error.endColumn) || ast[ast.length - 1]; + if (!commandContext) { + return []; + } + const possibleSolutions = await getSpellingPossibilities( + async () => + getCompatibleFunctionDefinitions(commandContext.name, undefined).concat( + // support nested expressions in STATS + commandContext.name === 'stats' ? getCompatibleFunctionDefinitions('eval', undefined) : [] + ), + errorText.substring(0, errorText.lastIndexOf('(')).toLowerCase() // reduce a bit the distance check making al lowercase + ); + return wrapIntoSpellingChangeAction( + error, + uri, + possibleSolutions.map((fn) => `${fn}${errorText.substring(errorText.lastIndexOf('('))}`) + ); +} + +function wrapIntoSpellingChangeAction( + error: monaco.editor.IMarkerData, + uri: monaco.Uri, + possibleSolution: string[] +): monaco.languages.CodeAction[] { + return possibleSolution.map((solution) => + createAction( + // @TODO: workout why the tooltip is truncating the title here + i18n.translate('monaco.esql.quickfix.replaceWithSolution', { + defaultMessage: 'Did you mean {solution} ?', + values: { + solution, + }, + }), + solution, + error, + uri + ) + ); +} + +function inferCodeFromError(error: monaco.editor.IMarkerData & { owner?: string }) { + if (error.message.includes('missing STRING')) { + const [, value] = error.message.split('at '); + return value.startsWith("'") && value.endsWith("'") ? 'wrongQuotes' : undefined; + } +} + +export async function getActions( + model: monaco.editor.ITextModel, + range: monaco.Range, + context: monaco.languages.CodeActionContext, + astProvider: AstProviderFn, + resourceRetriever?: ESQLCallbacks +): Promise { + const actions: monaco.languages.CodeAction[] = []; + if (context.markers.length === 0) { + return actions; + } + const innerText = model.getValue(); + const { ast } = await astProvider(innerText); + + const queryForFields = buildQueryForFieldsFromSource(innerText, ast); + const { getFieldsByType } = getFieldsByTypeRetriever(queryForFields, resourceRetriever); + const getSources = getSourcesRetriever(resourceRetriever); + const { getPolicies, getPolicyFields } = getPolicyRetriever(resourceRetriever); + + const callbacks = { + getFieldsByType, + getSources, + getPolicies, + getPolicyFields, + }; + + // Markers are sent only on hover and are limited to the hovered area + // so unless there are multiple error/markers for the same area, there's just one + // in some cases, like syntax + semantic errors (i.e. unquoted fields eval field-1 ), there might be more than one + for (const error of context.markers) { + const code = error.code ?? inferCodeFromError(error); + switch (code) { + case 'unknownColumn': + const [columnsSpellChanges, columnsQuotedChanges] = await Promise.all([ + getSpellingActionForColumns(error, model.uri, innerText, ast, callbacks), + getQuotableActionForColumns(error, model.uri, innerText, ast, callbacks), + ]); + actions.push(...(columnsQuotedChanges.length ? columnsQuotedChanges : columnsSpellChanges)); + break; + case 'unknownIndex': + const indexSpellChanges = await getSpellingActionForIndex( + error, + model.uri, + innerText, + ast, + callbacks + ); + actions.push(...indexSpellChanges); + break; + case 'unknownPolicy': + const policySpellChanges = await getSpellingActionForPolicies( + error, + model.uri, + innerText, + ast, + callbacks + ); + actions.push(...policySpellChanges); + break; + case 'unknownFunction': + const fnsSpellChanges = await getSpellingActionForFunctions( + error, + model.uri, + innerText, + ast + ); + actions.push(...fnsSpellChanges); + break; + case 'wrongQuotes': + // it is a syntax error, so location won't be helpful here + const [, errorText] = error.message.split('at '); + actions.push( + createAction( + i18n.translate('monaco.esql.quickfix.replaceWithQuote', { + defaultMessage: 'Change quote to " (double)', + }), + errorText.replaceAll("'", '"'), + // override the location + { ...error, endColumn: error.startColumn + errorText.length }, + model.uri + ) + ); + break; + default: + break; + } + } + return actions; +} diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/aggs.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/aggs.ts index 89d6e8a666d2d3..13e12f45049651 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/aggs.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/aggs.ts @@ -21,6 +21,7 @@ function createNumericAggDefinition({ const extraParamsExample = args.length ? `, ${args.map(({ value }) => value).join(',')}` : ''; return { name, + type: 'agg', description, supportedCommands: ['stats'], signatures: [ @@ -93,6 +94,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [ .concat([ { name: 'count', + type: 'agg', description: i18n.translate('monaco.esql.definitions.countDoc', { defaultMessage: 'Returns the count of the values in a field.', }), @@ -115,6 +117,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [ }, { name: 'count_distinct', + type: 'agg', description: i18n.translate('monaco.esql.definitions.countDistinctDoc', { defaultMessage: 'Returns the count of distinct values in a field.', }), @@ -132,6 +135,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [ }, { name: 'st_centroid', + type: 'agg', description: i18n.translate('monaco.esql.definitions.stCentroidDoc', { defaultMessage: 'Returns the count of distinct values in a field.', }), diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/builtin.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/builtin.ts index 8894e221143dd0..870c028835e1fa 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/builtin.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/builtin.ts @@ -14,9 +14,9 @@ function createMathDefinition( types: Array, description: string, warning?: FunctionDefinition['warning'] -) { +): FunctionDefinition { return { - builtin: true, + type: 'builtin', name, description, supportedCommands: ['eval', 'where', 'row'], @@ -52,9 +52,9 @@ function createComparisonDefinition( description: string; }, warning?: FunctionDefinition['warning'] -) { +): FunctionDefinition { return { - builtin: true, + type: 'builtin' as const, name, description, supportedCommands: ['eval', 'where', 'row'], @@ -113,18 +113,28 @@ export const builtinFunctions: FunctionDefinition[] = [ i18n.translate('monaco.esql.definition.divideDoc', { defaultMessage: 'Divide (/)', }), - (left, right) => { - if (right.type === 'literal' && right.literalType === 'number') { - return right.value === 0 - ? i18n.translate('monaco.esql.divide.warning.divideByZero', { - defaultMessage: 'Cannot divide by zero: {left}/{right}', - values: { - left: left.text, - right: right.value, - }, - }) - : undefined; + (fnDef) => { + const [left, right] = fnDef.args; + const messages = []; + if (!Array.isArray(left) && !Array.isArray(right)) { + if (right.type === 'literal' && right.literalType === 'number') { + if (right.value === 0) { + messages.push({ + type: 'warning' as const, + code: 'divideByZero', + text: i18n.translate('monaco.esql.divide.warning.divideByZero', { + defaultMessage: 'Cannot divide by zero: {left}/{right}', + values: { + left: left.text, + right: right.value, + }, + }), + location: fnDef.location, + }); + } + } } + return messages; } ), createMathDefinition( @@ -133,18 +143,28 @@ export const builtinFunctions: FunctionDefinition[] = [ i18n.translate('monaco.esql.definition.moduleDoc', { defaultMessage: 'Module (%)', }), - (left, right) => { - if (right.type === 'literal' && right.literalType === 'number') { - return right.value === 0 - ? i18n.translate('monaco.esql.divide.warning.zeroModule', { - defaultMessage: 'Module by zero can return null value: {left}/{right}', - values: { - left: left.text, - right: right.value, - }, - }) - : undefined; + (fnDef) => { + const [left, right] = fnDef.args; + const messages = []; + if (!Array.isArray(left) && !Array.isArray(right)) { + if (right.type === 'literal' && right.literalType === 'number') { + if (right.value === 0) { + messages.push({ + type: 'warning' as const, + code: 'moduleByZero', + text: i18n.translate('monaco.esql.divide.warning.zeroModule', { + defaultMessage: 'Module by zero can return null value: {left}/{right}', + values: { + left: left.text, + right: right.value, + }, + }), + location: fnDef.location, + }); + } + } } + return messages; } ), ...[ @@ -184,7 +204,7 @@ export const builtinFunctions: FunctionDefinition[] = [ defaultMessage: 'Greater than or equal to', }), }, - ].map((op) => createComparisonDefinition(op)), + ].map((op): FunctionDefinition => createComparisonDefinition(op)), ...[ // new special comparison operator for strings only { @@ -207,8 +227,8 @@ export const builtinFunctions: FunctionDefinition[] = [ }), }, { name: 'not_rlike', description: '' }, - ].map(({ name, description }) => ({ - builtin: true, + ].map(({ name, description }) => ({ + type: 'builtin' as const, ignoreAsSuggestion: /not/.test(name), name, description, @@ -233,8 +253,8 @@ export const builtinFunctions: FunctionDefinition[] = [ }), }, { name: 'not_in', description: '' }, - ].map(({ name, description }) => ({ - builtin: true, + ].map(({ name, description }) => ({ + type: 'builtin', ignoreAsSuggestion: /not/.test(name), name, description, @@ -284,7 +304,7 @@ export const builtinFunctions: FunctionDefinition[] = [ }), }, ].map(({ name, description }) => ({ - builtin: true, + type: 'builtin' as const, name, description, supportedCommands: ['eval', 'where', 'row'], @@ -300,7 +320,7 @@ export const builtinFunctions: FunctionDefinition[] = [ ], })), { - builtin: true, + type: 'builtin' as const, name: 'not', description: i18n.translate('monaco.esql.definition.notDoc', { defaultMessage: 'Not', @@ -315,7 +335,7 @@ export const builtinFunctions: FunctionDefinition[] = [ ], }, { - builtin: true, + type: 'builtin' as const, name: '=', description: i18n.translate('monaco.esql.definition.assignDoc', { defaultMessage: 'Assign (=)', @@ -334,6 +354,7 @@ export const builtinFunctions: FunctionDefinition[] = [ }, { name: 'functions', + type: 'builtin', description: i18n.translate('monaco.esql.definition.functionsDoc', { defaultMessage: 'Show ES|QL avaialble functions with signatures', }), @@ -347,6 +368,7 @@ export const builtinFunctions: FunctionDefinition[] = [ }, { name: 'info', + type: 'builtin', description: i18n.translate('monaco.esql.definition.infoDoc', { defaultMessage: 'Show information about the current ES node', }), diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts index 46b10a58375067..dcc9e6000d0090 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts @@ -146,6 +146,7 @@ export const commandDefinitions: CommandDefinition[] = [ defaultMessage: 'PROJECT command is no longer supported, please use KEEP instead', }), type: 'warning', + code: 'projectCommandDeprecated', }); } return messages; @@ -174,6 +175,7 @@ export const commandDefinitions: CommandDefinition[] = [ defaultMessage: 'Removing all fields is not allowed [*]', }), type: 'error' as const, + code: 'dropAllColumnsError', })) ); } @@ -187,6 +189,7 @@ export const commandDefinitions: CommandDefinition[] = [ defaultMessage: 'Drop [@timestamp] will remove all time filters to the search results', }), type: 'warning', + code: 'dropTimestampWarning', }); } return messages; @@ -317,6 +320,7 @@ export const commandDefinitions: CommandDefinition[] = [ }, }), type: 'warning' as const, + code: 'duplicateSettingWarning', })) ); } diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/functions.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/functions.ts index f7919061f423c5..a7da512c2a3dd0 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/functions.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/functions.ts @@ -1052,4 +1052,5 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [ ...def, supportedCommands: ['eval', 'where', 'row'], supportedOptions: ['by'], + type: 'eval', })); diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts index 1204741212cff2..76f3ca69b71127 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts @@ -99,6 +99,7 @@ export const appendSeparatorOption: CommandOptionsDefinition = { }, }), type: 'error', + code: 'wrongDissectOptionArgumentType', }); } return messages; diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts index 6d8aa34583031a..c6ea2229b230a9 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -import type { ESQLCommand, ESQLCommandOption, ESQLMessage, ESQLSingleAstItem } from '../types'; +import type { ESQLCommand, ESQLCommandOption, ESQLFunction, ESQLMessage } from '../types'; export interface FunctionDefinition { - builtin?: boolean; + type: 'builtin' | 'agg' | 'eval'; ignoreAsSuggestion?: boolean; name: string; alias?: string[]; @@ -29,7 +29,7 @@ export interface FunctionDefinition { returnType: string; examples?: string[]; }>; - warning?: (...args: ESQLSingleAstItem[]) => string | undefined; + warning?: (fnDef: ESQLFunction) => ESQLMessage[]; } export interface CommandBaseDefinition { diff --git a/packages/kbn-monaco/src/esql/lib/ast/shared/context.ts b/packages/kbn-monaco/src/esql/lib/ast/shared/context.ts index 5295b50b225250..aecb9afb97db99 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/shared/context.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/shared/context.ts @@ -132,7 +132,7 @@ function isNotEnrichClauseAssigment(node: ESQLFunction, command: ESQLCommand) { return node.name !== '=' && command.name !== 'enrich'; } function isBuiltinFunction(node: ESQLFunction) { - return Boolean(getFunctionDefinition(node.name)?.builtin); + return getFunctionDefinition(node.name)?.type === 'builtin'; } export function getAstContext(innerText: string, ast: ESQLAst, offset: number) { diff --git a/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts b/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts index 0056689b524e96..b0a72c817ac18e 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts @@ -166,6 +166,17 @@ export function isSupportedFunction( }; } +export function getAllFunctions(options?: { + type: Array | FunctionDefinition['type']; +}) { + const fns = buildFunctionLookup(); + if (!options?.type) { + return Array.from(fns.values()); + } + const types = new Set(Array.isArray(options.type) ? options.type : [options.type]); + return Array.from(fns.values()).filter((fn) => types.has(fn.type)); +} + export function getFunctionDefinition(name: string) { return buildFunctionLookup().get(name.toLowerCase()); } @@ -482,3 +493,10 @@ export function getLastCharFromTrimmed(text: string) { export function isRestartingExpression(text: string) { return getLastCharFromTrimmed(text) === ','; } + +export function shouldBeQuotedText( + text: string, + { dashSupported }: { dashSupported?: boolean } = {} +) { + return dashSupported ? /[^a-zA-Z\d_\.@-]/.test(text) : /[^a-zA-Z\d_\.@]/.test(text); +} diff --git a/packages/kbn-monaco/src/esql/lib/ast/shared/monaco_utils.ts b/packages/kbn-monaco/src/esql/lib/ast/shared/monaco_utils.ts new file mode 100644 index 00000000000000..4e7cf2cee7d8ab --- /dev/null +++ b/packages/kbn-monaco/src/esql/lib/ast/shared/monaco_utils.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EditorError } from '../../../../types'; +import { monaco } from '../../../../monaco_imports'; +import { ESQLMessage } from '../types'; + +// from linear offset to Monaco position +export function offsetToRowColumn(expression: string, offset: number): monaco.Position { + const lines = expression.split(/\n/); + let remainingChars = offset; + let lineNumber = 1; + for (const line of lines) { + if (line.length >= remainingChars) { + return new monaco.Position(lineNumber, remainingChars + 1); + } + remainingChars -= line.length + 1; + lineNumber++; + } + + throw new Error('Algorithm failure'); +} + +export function wrapAsMonacoMessage( + type: 'error' | 'warning', + code: string, + messages: Array +): EditorError[] { + const fallbackPosition = { column: 0, lineNumber: 0 }; + return messages.map((e) => { + if ('severity' in e) { + return e; + } + const startPosition = e.location ? offsetToRowColumn(code, e.location.min) : fallbackPosition; + const endPosition = e.location + ? offsetToRowColumn(code, e.location.max || 0) + : fallbackPosition; + return { + message: e.text, + startColumn: startPosition.column, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: type === 'error' ? monaco.MarkerSeverity.Error : monaco.MarkerSeverity.Warning, + _source: 'client' as const, + code: e.code, + }; + }); +} diff --git a/packages/kbn-monaco/src/esql/lib/ast/shared/resources_helpers.ts b/packages/kbn-monaco/src/esql/lib/ast/shared/resources_helpers.ts index fcd4cbb0737ffc..d4d5087e1464c3 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/shared/resources_helpers.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/shared/resources_helpers.ts @@ -8,6 +8,12 @@ import type { ESQLCallbacks } from './types'; import type { ESQLRealField } from '../validation/types'; +import { ESQLAst } from '../types'; + +export function buildQueryUntilPreviousCommand(ast: ESQLAst, queryString: string) { + const prevCommand = ast[Math.max(ast.length - 2, 0)]; + return prevCommand ? queryString.substring(0, prevCommand.location.max + 1) : queryString; +} export function getFieldsByTypeHelper(queryText: string, resourceRetriever?: ESQLCallbacks) { const cacheFields = new Map(); diff --git a/packages/kbn-monaco/src/esql/lib/ast/types.ts b/packages/kbn-monaco/src/esql/lib/ast/types.ts index b15c02baee5fe1..da5bff7551cada 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/types.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/types.ts @@ -85,6 +85,7 @@ export interface ESQLMessage { type: 'error' | 'warning'; text: string; location: ESQLLocation; + code: string; } export type AstProviderFn = ( diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts index b68e9aa987051d..f04d3d0eb9b224 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts @@ -232,13 +232,19 @@ export function getMessageFromId({ locations: ESQLLocation; }): ESQLMessage { const { message, type = 'error' } = getMessageAndTypeFromId(payload); - return createMessage(type, message, locations); + return createMessage(type, message, locations, payload.messageId); } -export function createMessage(type: 'error' | 'warning', message: string, location: ESQLLocation) { +export function createMessage( + type: 'error' | 'warning', + message: string, + location: ESQLLocation, + messageId: string +) { return { type, text: message, location, + code: messageId, }; } diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts index 18b83db16f234e..f8b1efe6fc53d0 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts @@ -51,7 +51,7 @@ import type { ESQLSingleAstItem, ESQLSource, } from '../types'; -import { getMessageFromId, createMessage } from './errors'; +import { getMessageFromId } from './errors'; import type { ESQLRealField, ESQLVariable, ReferenceMaps, ValidationResult } from './types'; import type { ESQLCallbacks } from '../shared/types'; import { @@ -325,11 +325,9 @@ function validateFunction( } // check if the definition has some warning to show: if (fnDefinition.warning) { - const message = fnDefinition.warning( - ...(astFunction.args.filter((arg) => !Array.isArray(arg)) as ESQLSingleAstItem[]) - ); - if (message) { - messages.push(createMessage('warning', message, astFunction.location)); + const payloads = fnDefinition.warning(astFunction); + if (payloads.length) { + messages.push(...payloads); } } // at this point we're sure that at least one signature is matching diff --git a/packages/kbn-monaco/src/esql/lib/monaco/esql_ast_provider.ts b/packages/kbn-monaco/src/esql/lib/monaco/esql_ast_provider.ts index 39bd193d5611ce..485c7d89ee8324 100644 --- a/packages/kbn-monaco/src/esql/lib/monaco/esql_ast_provider.ts +++ b/packages/kbn-monaco/src/esql/lib/monaco/esql_ast_provider.ts @@ -6,57 +6,15 @@ * Side Public License, v 1. */ -import type { EditorError } from '../../../types'; import type { ESQLCallbacks } from '../ast/shared/types'; import { monaco } from '../../../monaco_imports'; import type { ESQLWorker } from '../../worker/esql_worker'; import { suggest } from '../ast/autocomplete/autocomplete'; import { getHoverItem } from '../ast/hover'; import { getSignatureHelp } from '../ast/signature'; -import type { ESQLMessage } from '../ast/types'; import { validateAst } from '../ast/validation/validation'; - -// from linear offset to Monaco position -export function offsetToRowColumn(expression: string, offset: number): monaco.Position { - const lines = expression.split(/\n/); - let remainingChars = offset; - let lineNumber = 1; - for (const line of lines) { - if (line.length >= remainingChars) { - return new monaco.Position(lineNumber, remainingChars + 1); - } - remainingChars -= line.length + 1; - lineNumber++; - } - - throw new Error('Algorithm failure'); -} - -function wrapAsMonacoMessage( - type: 'error' | 'warning', - code: string, - messages: Array -): EditorError[] { - const fallbackPosition = { column: 0, lineNumber: 0 }; - return messages.map((e) => { - if ('severity' in e) { - return e; - } - const startPosition = e.location ? offsetToRowColumn(code, e.location.min) : fallbackPosition; - const endPosition = e.location - ? offsetToRowColumn(code, e.location.max || 0) - : fallbackPosition; - return { - message: e.text, - startColumn: startPosition.column, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: type === 'error' ? monaco.MarkerSeverity.Error : monaco.MarkerSeverity.Warning, - _source: 'client' as const, - }; - }); -} +import { getActions } from '../ast/code_actions'; +import { wrapAsMonacoMessage } from '../ast/shared/monaco_utils'; export class ESQLAstAdapter { constructor( @@ -118,4 +76,14 @@ export class ESQLAstAdapter { })), }; } + + async codeAction( + model: monaco.editor.ITextModel, + range: monaco.Range, + context: monaco.languages.CodeActionContext + ) { + const getAstFn = await this.getAstWorker(model); + const codeActions = await getActions(model, range, context, getAstFn, this.callbacks); + return codeActions; + } } diff --git a/packages/kbn-monaco/src/monaco_imports.ts b/packages/kbn-monaco/src/monaco_imports.ts index f6f89990995eb7..95f2fc44d857c4 100644 --- a/packages/kbn-monaco/src/monaco_imports.ts +++ b/packages/kbn-monaco/src/monaco_imports.ts @@ -23,6 +23,13 @@ import 'monaco-editor/esm/vs/editor/contrib/hover/browser/hover.js'; // Needed f import 'monaco-editor/esm/vs/editor/contrib/parameterHints/browser/parameterHints.js'; // Needed for signature import 'monaco-editor/esm/vs/editor/contrib/bracketMatching/browser/bracketMatching.js'; // Needed for brackets matching highlight +import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeAction.js'; +import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionCommands.js'; +import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionContributions.js'; +// import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionKeybindingResolver.js'; +import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionMenu.js'; +import 'monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionModel.js'; + import 'monaco-editor/esm/vs/language/json/monaco.contribution.js'; import 'monaco-editor/esm/vs/basic-languages/javascript/javascript.contribution.js'; // Needed for basic javascript support import 'monaco-editor/esm/vs/basic-languages/xml/xml.contribution.js'; // Needed for basic xml support diff --git a/packages/kbn-monaco/src/types.ts b/packages/kbn-monaco/src/types.ts index b2559fd919d160..e2268caab771fc 100644 --- a/packages/kbn-monaco/src/types.ts +++ b/packages/kbn-monaco/src/types.ts @@ -32,6 +32,7 @@ export interface LanguageProvidersModule { getSuggestionProvider: (callbacks?: Deps) => monaco.languages.CompletionItemProvider; getSignatureProvider?: (callbacks?: Deps) => monaco.languages.SignatureHelpProvider; getHoverProvider?: (callbacks?: Deps) => monaco.languages.HoverProvider; + getCodeActionProvider?: (callbacks?: Deps) => monaco.languages.CodeActionProvider; } export interface CustomLangModuleType @@ -47,6 +48,7 @@ export interface EditorError { endLineNumber: number; endColumn: number; message: string; + code?: string | undefined; } export interface LangValidation { diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx index 3546fcec41af40..1814a850648fc2 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx @@ -417,6 +417,11 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ [language, esqlCallbacks] ); + const codeActionProvider = useMemo( + () => (language === 'esql' ? ESQLLang.getCodeActionProvider?.(esqlCallbacks) : undefined), + [language, esqlCallbacks] + ); + const onErrorClick = useCallback(({ startLineNumber, startColumn }: MonacoMessage) => { if (!editor1.current) { return; @@ -541,6 +546,11 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ vertical: 'auto', }, overviewRulerBorder: false, + // this becomes confusing with multiple markers, so quick fixes + // will be proposed only within the tooltip + lightbulb: { + enabled: false, + }, readOnly: isLoading || isDisabled || @@ -776,6 +786,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ return hoverProvider?.provideHover(model, position, token); }, }} + codeActions={codeActionProvider} onChange={onQueryUpdate} editorDidMount={(editor) => { editor1.current = editor; diff --git a/packages/shared-ux/code_editor/impl/code_editor.tsx b/packages/shared-ux/code_editor/impl/code_editor.tsx index ca77fb3605f2f0..b41906d5ed4568 100644 --- a/packages/shared-ux/code_editor/impl/code_editor.tsx +++ b/packages/shared-ux/code_editor/impl/code_editor.tsx @@ -91,6 +91,13 @@ export interface CodeEditorProps { */ languageConfiguration?: monaco.languages.LanguageConfiguration; + /** + * CodeAction provider for code actions on markers feedback + * Documentation for the provider can be found here: + * https://microsoft.github.io/monaco-editor/docs.html#interfaces/languages.CodeActionProvider.html + */ + codeActions?: monaco.languages.CodeActionProvider; + /** * Function called before the editor is mounted in the view */ @@ -152,6 +159,7 @@ export const CodeEditor: React.FC = ({ hoverProvider, placeholder, languageConfiguration, + codeActions, 'aria-label': ariaLabel = i18n.translate('sharedUXPackages.codeEditor.ariaLabel', { defaultMessage: 'Code Editor', }), @@ -349,6 +357,10 @@ export const CodeEditor: React.FC = ({ if (languageConfiguration) { monaco.languages.setLanguageConfiguration(languageId, languageConfiguration); } + + if (codeActions) { + monaco.languages.registerCodeActionProvider(languageId, codeActions); + } }); // Register themes @@ -366,6 +378,7 @@ export const CodeEditor: React.FC = ({ suggestionProvider, signatureProvider, hoverProvider, + codeActions, languageConfiguration, ] ); From b6fbae2a2e76330b100d12a8f6bc3c48078979e3 Mon Sep 17 00:00:00 2001 From: Hannah Mudge Date: Wed, 31 Jan 2024 10:02:10 -0700 Subject: [PATCH 02/20] [Legacy Controls] Decouple `DEPRECATION_BADGE` action from Embeddable framework (#175928) Part of https://github.com/elastic/kibana/issues/175138 ## Summary This PR decouples the `DEPRECATION_BADGE` action (which is used specifically for the legacy input controls panel) from Embeddable framework by migrating to sets of composable interfaces. **Testing:** Since the legacy input control vis is deprecated and doesn't show up in the "Add panel" menu anymore, you need to navigate to `/app/visualize#/create?type=input_control_vis` in order to create this panel type for testing. ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../public/deprecation_badge.ts | 54 ++++++++++++++----- .../public/input_control_vis_type.ts | 4 +- src/plugins/input_control_vis/tsconfig.json | 1 + 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/plugins/input_control_vis/public/deprecation_badge.ts b/src/plugins/input_control_vis/public/deprecation_badge.ts index 5c65301066800b..d4da4709a0dc3f 100644 --- a/src/plugins/input_control_vis/public/deprecation_badge.ts +++ b/src/plugins/input_control_vis/public/deprecation_badge.ts @@ -6,19 +6,36 @@ * Side Public License, v 1. */ +import { ViewMode } from '@kbn/embeddable-plugin/public'; +import { i18n } from '@kbn/i18n'; +import { + apiCanAccessViewMode, + CanAccessViewMode, + EmbeddableApiContext, + getInheritedViewMode, + getViewModeSubject, +} from '@kbn/presentation-publishing'; import { Action } from '@kbn/ui-actions-plugin/public'; +import { apiHasVisualizeConfig, HasVisualizeConfig } from '@kbn/visualizations-plugin/public'; -import { Embeddable, ViewMode } from '@kbn/embeddable-plugin/public'; -import { i18n } from '@kbn/i18n'; -import { VisualizeInput } from '@kbn/visualizations-plugin/public'; +import { INPUT_CONTROL_VIS_TYPE } from './input_control_vis_type'; -export const ACTION_DEPRECATION_BADGE = 'ACTION_INPUT_CONTROL_DEPRECATION_BADGE'; +const ACTION_DEPRECATION_BADGE = 'ACTION_INPUT_CONTROL_DEPRECATION_BADGE'; -export interface DeprecationBadgeActionContext { - embeddable: Embeddable; -} +type InputControlDeprecationActionApi = CanAccessViewMode & HasVisualizeConfig; + +const isApiCompatible = (api: unknown | null): api is InputControlDeprecationActionApi => + Boolean(apiCanAccessViewMode(api) && apiHasVisualizeConfig(api)); + +const compatibilityCheck = (api: EmbeddableApiContext['embeddable']) => { + return ( + isApiCompatible(api) && + getInheritedViewMode(api) === ViewMode.EDIT && + api.getVis().type.name === INPUT_CONTROL_VIS_TYPE + ); +}; -export class InputControlDeprecationBadge implements Action { +export class InputControlDeprecationBadge implements Action { public id = ACTION_DEPRECATION_BADGE; public type = ACTION_DEPRECATION_BADGE; public disabled = true; @@ -40,11 +57,22 @@ export class InputControlDeprecationBadge implements Action) => void + ) { + if (!isApiCompatible(embeddable)) return; + return getViewModeSubject(embeddable)?.subscribe(() => { + onChange(compatibilityCheck(embeddable), this); + }); } public async execute() { diff --git a/src/plugins/input_control_vis/public/input_control_vis_type.ts b/src/plugins/input_control_vis/public/input_control_vis_type.ts index 875f87041e1e9d..1a4f2614b117e5 100644 --- a/src/plugins/input_control_vis/public/input_control_vis_type.ts +++ b/src/plugins/input_control_vis/public/input_control_vis_type.ts @@ -13,6 +13,8 @@ import { InputControlVisDependencies } from './plugin'; import { toExpressionAst } from './to_ast'; import { InputControlVisParams } from './types'; +export const INPUT_CONTROL_VIS_TYPE = 'input_control_vis'; + export function createInputControlVisTypeDefinition( deps: InputControlVisDependencies, readOnly: boolean @@ -20,7 +22,7 @@ export function createInputControlVisTypeDefinition( const ControlsTab = getControlsTab(deps); return { - name: 'input_control_vis', + name: INPUT_CONTROL_VIS_TYPE, title: i18n.translate('inputControl.register.controlsTitle', { defaultMessage: 'Input controls', }), diff --git a/src/plugins/input_control_vis/tsconfig.json b/src/plugins/input_control_vis/tsconfig.json index 876a37b925391f..5e6020990edd34 100644 --- a/src/plugins/input_control_vis/tsconfig.json +++ b/src/plugins/input_control_vis/tsconfig.json @@ -25,6 +25,7 @@ "@kbn/config-schema", "@kbn/ui-actions-plugin", "@kbn/embeddable-plugin", + "@kbn/presentation-publishing", ], "exclude": [ "target/**/*", From f3a7fd62e4e2b1d4260c4328f49effb1f8011c74 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Wed, 31 Jan 2024 18:07:18 +0100 Subject: [PATCH 03/20] Add criticality fields and risk score fields to alert schema (#174626) ## Update alerts fields names for asset criticality, and add risk score field We want to update `kibana.alert.user.criticality_level` to `host.asset.criticality` `kibana.alert.host.criticality_level` to `host.asset.criticality` `kibana.alert.user.criticality_level` and `kibana.alert.host.criticality_level` will be still present in the schema/mappings, for backward compatibility as it was released to serverless/ Also, we added `host.risk.calculated_score_norm`, `host.risk.calculated_level`, `user.risk.calculated_score_norm`, `user.risk.calculated_level`. Those fields enriched alerts from[8.5.0](https://github.com/elastic/kibana/pull/139478), but weren't added to the alert schema @SourinPaul [approved](https://github.com/elastic/kibana/pull/174626#issuecomment-1908721127) usage of new fields --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../src/schemas/generated/security_schema.ts | 2 + .../alert_as_data_fields.test.ts.snap | 80 +++++++++++++++++++ .../model/alerts/8.13.0/index.ts | 15 ++++ .../common/field_maps/8.13.0/alerts.ts | 17 +++- .../common/field_maps/field_names.ts | 19 ++++- .../alerts_table/default_config.test.tsx | 4 +- .../security_solution_detections/columns.ts | 6 +- .../rule_types/factories/utils/build_alert.ts | 12 +++ .../utils/enrichments/__mocks__/alerts.ts | 12 +++ .../enrichment_by_type/host_risk.ts | 9 ++- .../enrichment_by_type/user_risk.ts | 8 +- .../utils/enrichments/index.test.ts | 4 +- .../execution_logic/eql.ts | 2 +- .../execution_logic/esql.ts | 4 +- .../execution_logic/machine_learning.ts | 4 +- .../execution_logic/new_terms.ts | 4 +- .../execution_logic/query.ts | 8 +- .../execution_logic/threat_match.ts | 4 +- .../execution_logic/threshold.ts | 2 +- 19 files changed, 185 insertions(+), 31 deletions(-) diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts index fb9277abb884d6..bc8150356e039c 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts @@ -122,6 +122,7 @@ const SecurityAlertOptional = rt.partial({ 'ecs.version': schemaString, 'event.action': schemaString, 'event.kind': schemaString, + 'host.asset.criticality': schemaString, 'kibana.alert.action_group': schemaString, 'kibana.alert.ancestors.rule': schemaString, 'kibana.alert.building_block_type': schemaString, @@ -204,6 +205,7 @@ const SecurityAlertOptional = rt.partial({ 'kibana.alert.workflow_user': schemaString, 'kibana.version': schemaString, tags: schemaStringArray, + 'user.asset.criticality': schemaString, }); // prettier-ignore diff --git a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap index d009896a58505e..dab0c1c9a5737a 100644 --- a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap +++ b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap @@ -697,6 +697,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -1706,6 +1711,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -1734,6 +1744,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -2743,6 +2758,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -2771,6 +2791,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -3780,6 +3805,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -3808,6 +3838,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -4817,6 +4852,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -4845,6 +4885,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -5854,6 +5899,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -5888,6 +5938,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -6897,6 +6952,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -6925,6 +6985,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -7934,6 +7999,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; @@ -7962,6 +8032,11 @@ Object { "required": false, "type": "keyword", }, + "host.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.action_group": Object { "array": false, "required": false, @@ -8971,6 +9046,11 @@ Object { "required": false, "type": "keyword", }, + "user.asset.criticality": Object { + "array": false, + "required": false, + "type": "keyword", + }, }, } `; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.13.0/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.13.0/index.ts index 594dc685097db6..e7066058ab8db2 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.13.0/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.13.0/index.ts @@ -7,8 +7,14 @@ import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0'; import type { + LEGACY_ALERT_HOST_CRITICALITY, + LEGACY_ALERT_USER_CRITICALITY, ALERT_HOST_CRITICALITY, ALERT_USER_CRITICALITY, + ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM, + ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, + ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM, } from '../../../../../field_maps/field_names'; import type { Ancestor8120, @@ -29,8 +35,17 @@ new schemas to the union of all alert schemas, and re-export the new schemas as export type { Ancestor8120 as Ancestor8130 }; export interface BaseFields8130 extends BaseFields8120 { + [LEGACY_ALERT_HOST_CRITICALITY]: string | undefined; + [LEGACY_ALERT_USER_CRITICALITY]: string | undefined; [ALERT_HOST_CRITICALITY]: string | undefined; [ALERT_USER_CRITICALITY]: string | undefined; + /** + * Risk scores fields was added aroung 8.5.0, but the fields were not added to the alert schema + */ + [ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL]: string | undefined; + [ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM]: number | undefined; + [ALERT_USER_RISK_SCORE_CALCULATED_LEVEL]: string | undefined; + [ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM]: number | undefined; } export interface WrappedFields8130 { diff --git a/x-pack/plugins/security_solution/common/field_maps/8.13.0/alerts.ts b/x-pack/plugins/security_solution/common/field_maps/8.13.0/alerts.ts index 86c84092891b80..09bb86a2055002 100644 --- a/x-pack/plugins/security_solution/common/field_maps/8.13.0/alerts.ts +++ b/x-pack/plugins/security_solution/common/field_maps/8.13.0/alerts.ts @@ -6,10 +6,25 @@ */ import { alertsFieldMap840 } from '../8.4.0'; -import { ALERT_HOST_CRITICALITY, ALERT_USER_CRITICALITY } from '../field_names'; +import { + ALERT_HOST_CRITICALITY, + ALERT_USER_CRITICALITY, + LEGACY_ALERT_HOST_CRITICALITY, + LEGACY_ALERT_USER_CRITICALITY, +} from '../field_names'; export const alertsFieldMap8130 = { ...alertsFieldMap840, + [LEGACY_ALERT_HOST_CRITICALITY]: { + type: 'keyword', + array: false, + required: false, + }, + [LEGACY_ALERT_USER_CRITICALITY]: { + type: 'keyword', + array: false, + required: false, + }, /** * Stores the criticality level for the host, as determined by analysts, in relation to the alert. * The Criticality level is copied from the asset criticality index. diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index 6124cc08ebd2bb..b8ef2e61fb3900 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -17,8 +17,23 @@ export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as c export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; export const ALERT_NEW_TERMS_FIELDS = `${ALERT_RULE_PARAMETERS}.new_terms_fields` as const; -export const ALERT_HOST_CRITICALITY = `${ALERT_NAMESPACE}.host.criticality_level` as const; -export const ALERT_USER_CRITICALITY = `${ALERT_NAMESPACE}.user.criticality_level` as const; +/** + * @deprecated Use {@link ALERT_HOST_CRITICALITY} + */ +export const LEGACY_ALERT_HOST_CRITICALITY = `${ALERT_NAMESPACE}.host.criticality_level` as const; +/** + * @deprecated Use {@link ALERT_USER_CRITICALITY} + */ +export const LEGACY_ALERT_USER_CRITICALITY = `${ALERT_NAMESPACE}.user.criticality_level` as const; + +export const ALERT_HOST_CRITICALITY = `host.asset.criticality` as const; +export const ALERT_USER_CRITICALITY = `user.asset.criticality` as const; +export const ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL = `host.risk.calculated_level` as const; +export const ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM = + `host.risk.calculated_score_norm` as const; +export const ALERT_USER_RISK_SCORE_CALCULATED_LEVEL = `user.risk.calculated_level` as const; +export const ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM = + `user.risk.calculated_score_norm` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx index fc5d91bcaf8e5b..1e0c7021929c9c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx @@ -70,8 +70,8 @@ const platinumBaseColumns = [ { columnHeaderType: 'not-filtered', id: 'host.risk.calculated_level' }, { columnHeaderType: 'not-filtered', id: 'user.name' }, { columnHeaderType: 'not-filtered', id: 'user.risk.calculated_level' }, - { columnHeaderType: 'not-filtered', id: 'kibana.alert.host.criticality_level' }, - { columnHeaderType: 'not-filtered', id: 'kibana.alert.user.criticality_level' }, + { columnHeaderType: 'not-filtered', id: 'host.asset.criticality' }, + { columnHeaderType: 'not-filtered', id: 'user.asset.criticality' }, { columnHeaderType: 'not-filtered', id: 'process.name' }, { columnHeaderType: 'not-filtered', id: 'file.name' }, { columnHeaderType: 'not-filtered', id: 'source.ip' }, diff --git a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts index b22bf5e2ed4294..fc3f5afa897a2c 100644 --- a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts +++ b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts @@ -9,6 +9,8 @@ import type { EuiDataGridColumn } from '@elastic/eui'; import { ALERT_HOST_CRITICALITY, ALERT_USER_CRITICALITY, + ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, } from '../../../../common/field_maps/field_names'; import type { LicenseService } from '../../../../common/license'; import type { ColumnHeaderOptions } from '../../../../common/types'; @@ -64,7 +66,7 @@ const getBaseColumns = ( isPlatinumPlus ? { columnHeaderType: defaultColumnHeaderType, - id: 'host.risk.calculated_level', + id: ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, } : null, { @@ -74,7 +76,7 @@ const getBaseColumns = ( isPlatinumPlus ? { columnHeaderType: defaultColumnHeaderType, - id: 'user.risk.calculated_level', + id: ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, } : null, isPlatinumPlus diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index 024f1b123ff997..d81fe7d0202829 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -80,8 +80,14 @@ import { ALERT_RULE_THREAT, ALERT_RULE_EXCEPTIONS_LIST, ALERT_RULE_IMMUTABLE, + LEGACY_ALERT_HOST_CRITICALITY, + LEGACY_ALERT_USER_CRITICALITY, ALERT_HOST_CRITICALITY, ALERT_USER_CRITICALITY, + ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM, + ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, + ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM, } from '../../../../../../common/field_maps/field_names'; import type { CompleteRule, RuleParams } from '../../../rule_schema'; import { commonParamsCamelToSnake, typeSpecificCamelToSnake } from '../../../rule_management'; @@ -259,8 +265,14 @@ export const buildAlert = ( 'kibana.alert.rule.severity': params.severity, 'kibana.alert.rule.building_block_type': params.buildingBlockType, // asset criticality fields will be enriched before ingestion + [LEGACY_ALERT_HOST_CRITICALITY]: undefined, + [LEGACY_ALERT_USER_CRITICALITY]: undefined, [ALERT_HOST_CRITICALITY]: undefined, [ALERT_USER_CRITICALITY]: undefined, + [ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL]: undefined, + [ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, + [ALERT_USER_RISK_SCORE_CALCULATED_LEVEL]: undefined, + [ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts index efbf39d815aeac..218ba29bfdc45a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts @@ -70,6 +70,12 @@ import { ALERT_RULE_TIMESTAMP_OVERRIDE, ALERT_HOST_CRITICALITY, ALERT_USER_CRITICALITY, + LEGACY_ALERT_HOST_CRITICALITY, + LEGACY_ALERT_USER_CRITICALITY, + ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM, + ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, + ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM, } from '../../../../../../../common/field_maps/field_names'; export const createAlert = ( @@ -196,8 +202,14 @@ export const createAlert = ( rule_name_override: undefined, timestamp_override: undefined, }, + [LEGACY_ALERT_HOST_CRITICALITY]: undefined, + [LEGACY_ALERT_USER_CRITICALITY]: undefined, [ALERT_HOST_CRITICALITY]: undefined, [ALERT_USER_CRITICALITY]: undefined, + [ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL]: undefined, + [ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, + [ALERT_USER_RISK_SCORE_CALCULATED_LEVEL]: undefined, + [ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, ...data, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/host_risk.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/host_risk.ts index 1b34f6cb878595..7599cf28e9183c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/host_risk.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/host_risk.ts @@ -6,7 +6,10 @@ */ import { set } from '@kbn/safer-lodash-set'; import { cloneDeep } from 'lodash'; - +import { + ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM, +} from '../../../../../../../common/field_maps/field_names'; import { getHostRiskIndex } from '../../../../../../../common/search_strategy/security_solution/risk_score/common'; import { RiskScoreFields } from '../../../../../../../common/search_strategy/security_solution/risk_score/all'; import { createSingleFieldMatchEnrichment } from '../create_single_field_match_enrichment'; @@ -43,10 +46,10 @@ export const createHostRiskEnrichments: CreateRiskEnrichment = async ({ } const newEvent = cloneDeep(event); if (riskLevel) { - set(newEvent, '_source.host.risk.calculated_level', riskLevel); + set(newEvent, `_source.${ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL}`, riskLevel); } if (riskScore) { - set(newEvent, '_source.host.risk.calculated_score_norm', riskScore); + set(newEvent, `_source.${ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM}`, riskScore); } return newEvent; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/user_risk.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/user_risk.ts index 27ae894f281343..ad4ca20b140ac8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/user_risk.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/enrichment_by_type/user_risk.ts @@ -6,6 +6,10 @@ */ import { set } from '@kbn/safer-lodash-set'; import { cloneDeep } from 'lodash'; +import { + ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, + ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM, +} from '../../../../../../../common/field_maps/field_names'; import { getUserRiskIndex } from '../../../../../../../common/search_strategy/security_solution/risk_score/common'; import { RiskScoreFields } from '../../../../../../../common/search_strategy/security_solution/risk_score/all'; import { createSingleFieldMatchEnrichment } from '../create_single_field_match_enrichment'; @@ -42,10 +46,10 @@ export const createUserRiskEnrichments: CreateRiskEnrichment = async ({ } const newEvent = cloneDeep(event); if (riskLevel) { - set(newEvent, '_source.user.risk.calculated_level', riskLevel); + set(newEvent, `_source.${ALERT_USER_RISK_SCORE_CALCULATED_LEVEL}`, riskLevel); } if (riskScore) { - set(newEvent, '_source.user.risk.calculated_score_norm', riskScore); + set(newEvent, `_source.${ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM}`, riskScore); } return newEvent; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/index.test.ts index 4c87c6f5a82720..ac3368e9fa7fbd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/index.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/index.test.ts @@ -211,8 +211,8 @@ describe('enrichEvents', () => { ...createEntity('user', 'user name 1'), ...createEntity('host', 'host name 1'), - 'kibana.alert.host.criticality_level': 'low', - 'kibana.alert.user.criticality_level': 'important', + 'host.asset.criticality': 'low', + 'user.asset.criticality': 'important', }), createAlert('2', { ...createEntity('host', 'user name 1'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index 0eddd53de4c909..77d8b5ffb373d6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -646,7 +646,7 @@ export default ({ getService }: FtrProviderContext) => { const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); const fullAlert = previewAlerts[0]._source; - expect(fullAlert?.['kibana.alert.host.criticality_level']).to.eql('important'); + expect(fullAlert?.['host.asset.criticality']).to.eql('important'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index 643728fbd27c16..c582b9d94f3ce8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -892,9 +892,7 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).toBe(1); - expect(previewAlerts[0]?._source?.['kibana.alert.host.criticality_level']).toBe( - 'very_important' - ); + expect(previewAlerts[0]?._source?.['host.asset.criticality']).toBe('very_important'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts index a7d12cc0482441..e222f1ddd7cb4d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts @@ -289,8 +289,8 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).toBe(1); const fullAlert = previewAlerts[0]._source; - expect(fullAlert?.['kibana.alert.host.criticality_level']).toBe('normal'); - expect(fullAlert?.['kibana.alert.user.criticality_level']).toBe('very_important'); + expect(fullAlert?.['host.asset.criticality']).toBe('normal'); + expect(fullAlert?.['user.asset.criticality']).toBe('very_important'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts index 5d1726e0cc1c73..7cacab1066da44 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts @@ -1062,8 +1062,8 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId }); const fullAlert = previewAlerts[0]._source; - expect(fullAlert?.['kibana.alert.host.criticality_level']).to.eql('normal'); - expect(fullAlert?.['kibana.alert.user.criticality_level']).to.eql('very_important'); + expect(fullAlert?.['host.asset.criticality']).to.eql('normal'); + expect(fullAlert?.['user.asset.criticality']).to.eql('very_important'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/query.ts index 060dfff2b20bc1..ac7aa41223c9e0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/query.ts @@ -296,12 +296,8 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts[0]?._source?.['kibana.alert.host.criticality_level']).to.eql( - 'important' - ); - expect(previewAlerts[0]?._source?.['kibana.alert.user.criticality_level']).to.eql( - 'very_important' - ); + expect(previewAlerts[0]?._source?.['host.asset.criticality']).to.eql('important'); + expect(previewAlerts[0]?._source?.['user.asset.criticality']).to.eql('very_important'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match.ts index 60215e58030dc4..e8cbeb2c1b4b3b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match.ts @@ -1663,8 +1663,8 @@ export default ({ getService }: FtrProviderContext) => { return expect(fullAlert).to.be.ok(); } - expect(fullAlert?.['kibana.alert.host.criticality_level']).to.eql('low'); - expect(fullAlert?.['kibana.alert.user.criticality_level']).to.eql('very_important'); + expect(fullAlert?.['host.asset.criticality']).to.eql('low'); + expect(fullAlert?.['user.asset.criticality']).to.eql('very_important'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts index 71f25f48345b44..9449750b384656 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts @@ -452,7 +452,7 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['host.name'] }); const fullAlert = previewAlerts[0]?._source; - expect(fullAlert?.['kibana.alert.host.criticality_level']).toEqual('important'); + expect(fullAlert?.['host.asset.criticality']).toEqual('important'); }); }); }); From 345c2cb2d99f9a1c3423654921f4101c55b522a3 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 17:08:55 +0000 Subject: [PATCH 04/20] skip flaky suite (#169785) --- .../osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts index 041818824f2f23..b45024add798c2 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts @@ -24,7 +24,8 @@ import { import { clickRuleName, inputQuery, typeInECSFieldInput } from '../../tasks/live_query'; import { closeDateTabIfVisible, closeToastIfVisible } from '../../tasks/integrations'; -describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/169785 +describe.skip('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serverless'] }, () => { let multiQueryPackId: string; let multiQueryPackName: string; let ruleId: string; From d3a267e5141efae8b716bbf7b093a32b0cbeafb1 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 17:10:25 +0000 Subject: [PATCH 05/20] skip flaky suite (#175956) --- .../cases/public/components/files/file_delete_button.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cases/public/components/files/file_delete_button.test.tsx b/x-pack/plugins/cases/public/components/files/file_delete_button.test.tsx index 38ed8a20eab40f..8c4540aaadbec1 100644 --- a/x-pack/plugins/cases/public/components/files/file_delete_button.test.tsx +++ b/x-pack/plugins/cases/public/components/files/file_delete_button.test.tsx @@ -21,7 +21,8 @@ jest.mock('../../containers/use_delete_file_attachment'); const useDeleteFileAttachmentMock = useDeleteFileAttachment as jest.Mock; -describe('FileDeleteButton', () => { +// FLAKY: https://github.com/elastic/kibana/issues/175956 +describe.skip('FileDeleteButton', () => { let appMockRender: AppMockRenderer; const mutate = jest.fn(); From b162020e75c5e8354d3e497627ad2d42578a3ccb Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 17:11:34 +0000 Subject: [PATCH 06/20] skip flaky suite (#104578) --- x-pack/test/functional/apps/discover/saved_searches.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/discover/saved_searches.ts b/x-pack/test/functional/apps/discover/saved_searches.ts index 8f5fe5dc9bc11a..c95249e927084e 100644 --- a/x-pack/test/functional/apps/discover/saved_searches.ts +++ b/x-pack/test/functional/apps/discover/saved_searches.ts @@ -45,7 +45,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.common.unsetTime(); }); - describe('Customize time range', () => { + // FLAKY: https://github.com/elastic/kibana/issues/104578 + describe.skip('Customize time range', () => { it('should be possible to customize time range for saved searches on dashboards', async () => { await PageObjects.dashboard.navigateToApp(); await PageObjects.dashboard.clickNewDashboard(); From 5b83fe3899cd7c444c6586da7a27c270b2fb6e98 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 17:12:51 +0000 Subject: [PATCH 07/20] skip flaky suite (#172781) --- .../examples/unified_field_list_examples/existing_fields.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples/existing_fields.ts b/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples/existing_fields.ts index 6dd1542bbea394..1b3bf8a63caa26 100644 --- a/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples/existing_fields.ts +++ b/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples/existing_fields.ts @@ -93,7 +93,8 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await PageObjects.svlCommonPage.forceLogout(); }); - describe('existence', () => { + // FLAKY: https://github.com/elastic/kibana/issues/172781 + describe.skip('existence', () => { it('should find which fields exist in the sample documents', async () => { const sidebarFields = await PageObjects.unifiedFieldList.getAllFieldNames(); expect(sidebarFields.sort()).to.eql([...metaFields, ...fieldsWithData].sort()); From f0613cb46a8f0f6f1c7e900c21268f7a906fe45f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 17:25:06 +0000 Subject: [PATCH 08/20] skip flaky suite (#170118) --- .../public/resolver/view/clickthrough.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx index ffebf2e1d780a5..2bc671c1b7d780 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx @@ -439,7 +439,8 @@ describe('Resolver, when analyzing a tree that has 2 related registry and 1 rela }); }); - describe('when it has loaded', () => { + // FLAKY: https://github.com/elastic/kibana/issues/170118 + describe.skip('when it has loaded', () => { let originBounds: AABB; let firstChildBounds: AABB; let secondChildBounds: AABB; From be645db0c718163608080b283fc78a0c564980d0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 31 Jan 2024 11:42:10 -0600 Subject: [PATCH 09/20] fix lint error --- .../all/alerts_response_actions_form.cy.ts | 424 +++++++++--------- 1 file changed, 214 insertions(+), 210 deletions(-) diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts index b45024add798c2..45d87f0766633d 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts @@ -25,223 +25,227 @@ import { clickRuleName, inputQuery, typeInECSFieldInput } from '../../tasks/live import { closeDateTabIfVisible, closeToastIfVisible } from '../../tasks/integrations'; // FLAKY: https://github.com/elastic/kibana/issues/169785 -describe.skip('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serverless'] }, () => { - let multiQueryPackId: string; - let multiQueryPackName: string; - let ruleId: string; - let ruleName: string; - let packId: string; - let packName: string; - const packData = packFixture(); - const multiQueryPackData = multiQueryPackFixture(); - before(() => { - initializeDataViews(); - }); - beforeEach(() => { - loadPack(packData).then((data) => { - packId = data.saved_object_id; - packName = data.name; - }); - loadPack(multiQueryPackData).then((data) => { - multiQueryPackId = data.saved_object_id; - multiQueryPackName = data.name; - }); - loadRule().then((data) => { - ruleId = data.id; - ruleName = data.name; - }); - }); - afterEach(() => { - cleanupPack(packId); - cleanupPack(multiQueryPackId); - cleanupRule(ruleId); - }); - - it('adds response actions with osquery with proper validation and form values', () => { - cy.visit('/app/security/rules'); - clickRuleName(ruleName); - cy.getBySel('globalLoadingIndicator').should('not.exist'); - cy.getBySel('editRuleSettingsLink').click(); - cy.getBySel('globalLoadingIndicator').should('not.exist'); - closeDateTabIfVisible(); - cy.getBySel('edit-rule-actions-tab').click(); - cy.getBySel('globalLoadingIndicator').should('not.exist'); - cy.contains('Response actions are run on each rule execution.'); - cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); - - cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { - cy.contains('Query is a required field'); - cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); - }); - - // check if changing error state of one input doesn't clear other errors - START - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.contains('Advanced').click(); - cy.getBySel('timeout-input').clear(); - cy.contains('The timeout value must be 60 seconds or higher.'); - }); - - cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { - cy.contains('Query is a required field'); - cy.contains('The timeout value must be 60 seconds or higher.'); - }); - - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.getBySel('timeout-input').type('6'); - cy.contains('The timeout value must be 60 seconds or higher.'); - }); - cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { - cy.contains('Query is a required field'); - cy.contains('The timeout value must be 60 seconds or higher.'); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.getBySel('timeout-input').type('6'); - cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); - }); - cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { - cy.contains('Query is a required field'); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.getBySel('timeout-input').type('6'); - }); - cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { - cy.contains('Query is a required field'); - cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); - }); - // check if changing error state of one input doesn't clear other errors - END +describe.skip( + 'Alert Event Details - Response Actions Form', + { tags: ['@ess', '@serverless'] }, + () => { + let multiQueryPackId: string; + let multiQueryPackName: string; + let ruleId: string; + let ruleName: string; + let packId: string; + let packName: string; + const packData = packFixture(); + const multiQueryPackData = multiQueryPackFixture(); + before(() => { + initializeDataViews(); + }); + beforeEach(() => { + loadPack(packData).then((data) => { + packId = data.saved_object_id; + packName = data.name; + }); + loadPack(multiQueryPackData).then((data) => { + multiQueryPackId = data.saved_object_id; + multiQueryPackName = data.name; + }); + loadRule().then((data) => { + ruleId = data.id; + ruleName = data.name; + }); + }); + afterEach(() => { + cleanupPack(packId); + cleanupPack(multiQueryPackId); + cleanupRule(ruleId); + }); + + it('adds response actions with osquery with proper validation and form values', () => { + cy.visit('/app/security/rules'); + clickRuleName(ruleName); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.getBySel('editRuleSettingsLink').click(); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + closeDateTabIfVisible(); + cy.getBySel('edit-rule-actions-tab').click(); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.contains('Response actions are run on each rule execution.'); + cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); + + cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { + cy.contains('Query is a required field'); + cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); + }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.contains('Query is a required field'); - inputQuery('select * from uptime1'); - }); - cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); - cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { - cy.contains('Run a set of queries in a pack').click(); - }); - cy.getBySel(RESPONSE_ACTIONS_ERRORS) - .within(() => { - cy.contains('Pack is a required field'); - }) - .should('exist'); - cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { - cy.contains('Pack is a required field'); - cy.getBySel('comboBoxInput').type(`${packName}{downArrow}{enter}`); - }); + // check if changing error state of one input doesn't clear other errors - START + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.contains('Advanced').click(); + cy.getBySel('timeout-input').clear(); + cy.contains('The timeout value must be 60 seconds or higher.'); + }); - cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); + cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { + cy.contains('Query is a required field'); + cy.contains('The timeout value must be 60 seconds or higher.'); + }); + + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.getBySel('timeout-input').type('6'); + cy.contains('The timeout value must be 60 seconds or higher.'); + }); + cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { + cy.contains('Query is a required field'); + cy.contains('The timeout value must be 60 seconds or higher.'); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.getBySel('timeout-input').type('6'); + cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); + }); + cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { + cy.contains('Query is a required field'); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.getBySel('timeout-input').type('6'); + }); + cy.getBySel(RESPONSE_ACTIONS_ERRORS).within(() => { + cy.contains('Query is a required field'); + cy.contains('The timeout value must be 60 seconds or higher.').should('not.exist'); + }); + // check if changing error state of one input doesn't clear other errors - END - cy.getBySel(RESPONSE_ACTIONS_ITEM_2) - .within(() => { + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { cy.contains('Query is a required field'); - inputQuery('select * from uptime'); - cy.contains('Query is a required field').should('not.exist'); - cy.contains('Advanced').click(); - typeInECSFieldInput('{downArrow}{enter}'); - cy.getBySel('osqueryColumnValueSelect').type('days{downArrow}{enter}'); - }) - .clickOutside(); - - cy.getBySel('ruleEditSubmitButton').click(); - cy.contains(`${ruleName} was saved`).should('exist'); - closeToastIfVisible(); - - cy.getBySel('globalLoadingIndicator').should('not.exist'); - cy.getBySel('editRuleSettingsLink').click(); - cy.getBySel('globalLoadingIndicator').should('not.exist'); - cy.getBySel('edit-rule-actions-tab').click(); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.contains('select * from uptime1'); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_2).within(() => { - cy.contains('select * from uptime'); - cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}'); - cy.contains('Days of uptime'); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { - cy.getBySel('comboBoxSearchInput').should('have.value', packName); - cy.getBySel('comboBoxInput').type('{selectall}{backspace}{enter}'); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { - cy.contains('select * from uptime1'); - cy.getBySel('remove-response-action').click(); - }); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0) - .within(() => { - cy.getBySel('comboBoxSearchInput').click(); - cy.contains('Search for a pack to run'); + inputQuery('select * from uptime1'); + }); + cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); + cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { + cy.contains('Run a set of queries in a pack').click(); + }); + cy.getBySel(RESPONSE_ACTIONS_ERRORS) + .within(() => { + cy.contains('Pack is a required field'); + }) + .should('exist'); + cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { cy.contains('Pack is a required field'); cy.getBySel('comboBoxInput').type(`${packName}{downArrow}{enter}`); - }) - .clickOutside(); - cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { - cy.contains('select * from uptime'); - cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}'); - cy.contains('Days of uptime'); - }); - - cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleSingleQuery'); - - cy.getBySel('ruleEditSubmitButton').click(); - cy.wait('@saveRuleSingleQuery').should(({ request }) => { - const oneQuery = [ - { - interval: 3600, - query: 'select * from uptime;', - id: Object.keys(packData.queries)[0], - }, - ]; - expect(request.body.response_actions[0].params.queries).to.deep.equal(oneQuery); - }); - - cy.contains(`${ruleName} was saved`).should('exist'); - closeToastIfVisible(); - - cy.getBySel('globalLoadingIndicator').should('not.exist'); - cy.getBySel('editRuleSettingsLink').click(); - cy.getBySel('globalLoadingIndicator').should('not.exist'); - - cy.getBySel('edit-rule-actions-tab').click(); - cy.getBySel(RESPONSE_ACTIONS_ITEM_0) - .within(() => { + }); + + cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click(); + + cy.getBySel(RESPONSE_ACTIONS_ITEM_2) + .within(() => { + cy.contains('Query is a required field'); + inputQuery('select * from uptime'); + cy.contains('Query is a required field').should('not.exist'); + cy.contains('Advanced').click(); + typeInECSFieldInput('{downArrow}{enter}'); + cy.getBySel('osqueryColumnValueSelect').type('days{downArrow}{enter}'); + }) + .clickOutside(); + + cy.getBySel('ruleEditSubmitButton').click(); + cy.contains(`${ruleName} was saved`).should('exist'); + closeToastIfVisible(); + + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.getBySel('editRuleSettingsLink').click(); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.getBySel('edit-rule-actions-tab').click(); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.contains('select * from uptime1'); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_2).within(() => { + cy.contains('select * from uptime'); + cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}'); + cy.contains('Days of uptime'); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { cy.getBySel('comboBoxSearchInput').should('have.value', packName); - cy.getBySel('comboBoxInput').type( - `{selectall}{backspace}${multiQueryPackName}{downArrow}{enter}` - ); - cy.contains('SELECT * FROM memory_info;'); - cy.contains('SELECT * FROM system_info;'); - }) - .clickOutside(); - - cy.getBySel(RESPONSE_ACTIONS_ITEM_1) - .within(() => { + cy.getBySel('comboBoxInput').type('{selectall}{backspace}{enter}'); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => { + cy.contains('select * from uptime1'); + cy.getBySel('remove-response-action').click(); + }); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0) + .within(() => { + cy.getBySel('comboBoxSearchInput').click(); + cy.contains('Search for a pack to run'); + cy.contains('Pack is a required field'); + cy.getBySel('comboBoxInput').type(`${packName}{downArrow}{enter}`); + }) + .clickOutside(); + cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { cy.contains('select * from uptime'); cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}'); cy.contains('Days of uptime'); - }) - .clickOutside(); - cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleMultiQuery'); - - cy.contains('Save changes').click(); - cy.wait('@saveRuleMultiQuery').should(({ request }) => { - const threeQueries = [ - { - interval: 3600, - query: 'SELECT * FROM memory_info;', - platform: 'linux', - id: Object.keys(multiQueryPackData.queries)[0], - }, - { - interval: 3600, - query: 'SELECT * FROM system_info;', - id: Object.keys(multiQueryPackData.queries)[1], - }, - { - interval: 10, - query: 'select opera_extensions.* from users join opera_extensions using (uid);', - id: Object.keys(multiQueryPackData.queries)[2], - }, - ]; - expect(request.body.response_actions[0].params.queries).to.deep.equal(threeQueries); - }); - }); -}); + }); + + cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleSingleQuery'); + + cy.getBySel('ruleEditSubmitButton').click(); + cy.wait('@saveRuleSingleQuery').should(({ request }) => { + const oneQuery = [ + { + interval: 3600, + query: 'select * from uptime;', + id: Object.keys(packData.queries)[0], + }, + ]; + expect(request.body.response_actions[0].params.queries).to.deep.equal(oneQuery); + }); + + cy.contains(`${ruleName} was saved`).should('exist'); + closeToastIfVisible(); + + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.getBySel('editRuleSettingsLink').click(); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + + cy.getBySel('edit-rule-actions-tab').click(); + cy.getBySel(RESPONSE_ACTIONS_ITEM_0) + .within(() => { + cy.getBySel('comboBoxSearchInput').should('have.value', packName); + cy.getBySel('comboBoxInput').type( + `{selectall}{backspace}${multiQueryPackName}{downArrow}{enter}` + ); + cy.contains('SELECT * FROM memory_info;'); + cy.contains('SELECT * FROM system_info;'); + }) + .clickOutside(); + + cy.getBySel(RESPONSE_ACTIONS_ITEM_1) + .within(() => { + cy.contains('select * from uptime'); + cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}'); + cy.contains('Days of uptime'); + }) + .clickOutside(); + cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleMultiQuery'); + + cy.contains('Save changes').click(); + cy.wait('@saveRuleMultiQuery').should(({ request }) => { + const threeQueries = [ + { + interval: 3600, + query: 'SELECT * FROM memory_info;', + platform: 'linux', + id: Object.keys(multiQueryPackData.queries)[0], + }, + { + interval: 3600, + query: 'SELECT * FROM system_info;', + id: Object.keys(multiQueryPackData.queries)[1], + }, + { + interval: 10, + query: 'select opera_extensions.* from users join opera_extensions using (uid);', + id: Object.keys(multiQueryPackData.queries)[2], + }, + ]; + expect(request.body.response_actions[0].params.queries).to.deep.equal(threeQueries); + }); + }); + } +); From 4a92a8832f2df2e97a18b12b6ff1cdb2d3e1ce5d Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Wed, 31 Jan 2024 19:56:24 +0200 Subject: [PATCH 10/20] [ES|QL] Use same adhoc dataviews for queries with the same index pattern (#174736) --- .github/CODEOWNERS | 1 + package.json | 1 + packages/kbn-es-query/tsconfig.json | 2 +- packages/kbn-esql-utils/README.md | 4 ++ packages/kbn-esql-utils/index.ts | 9 +++++ packages/kbn-esql-utils/jest.config.js | 13 +++++++ packages/kbn-esql-utils/kibana.jsonc | 5 +++ packages/kbn-esql-utils/package.json | 7 ++++ packages/kbn-esql-utils/src/index.ts | 9 +++++ packages/kbn-esql-utils/src/utils/index.ts | 37 +++++++++++++++++++ packages/kbn-esql-utils/src/utils/sha256.ts | 11 ++++++ packages/kbn-esql-utils/tsconfig.json | 22 +++++++++++ .../get_data_view_by_text_based_query_lang.ts | 5 +-- src/plugins/discover/tsconfig.json | 1 + tsconfig.base.json | 2 + .../shared/edit_on_the_fly/helpers.ts | 10 ++--- .../public/datasources/text_based/utils.ts | 8 ++-- x-pack/plugins/lens/tsconfig.json | 1 + yarn.lock | 4 ++ 19 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 packages/kbn-esql-utils/README.md create mode 100644 packages/kbn-esql-utils/index.ts create mode 100644 packages/kbn-esql-utils/jest.config.js create mode 100644 packages/kbn-esql-utils/kibana.jsonc create mode 100644 packages/kbn-esql-utils/package.json create mode 100644 packages/kbn-esql-utils/src/index.ts create mode 100644 packages/kbn-esql-utils/src/utils/index.ts create mode 100644 packages/kbn-esql-utils/src/utils/sha256.ts create mode 100644 packages/kbn-esql-utils/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b1a14c058fa956..b120feb392a1c9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -383,6 +383,7 @@ packages/kbn-eslint-plugin-imports @elastic/kibana-operations packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team examples/eso_model_version_example @elastic/kibana-security x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security +packages/kbn-esql-utils @elastic/kibana-visualizations packages/kbn-event-annotation-common @elastic/kibana-visualizations packages/kbn-event-annotation-components @elastic/kibana-visualizations src/plugins/event_annotation_listing @elastic/kibana-visualizations diff --git a/package.json b/package.json index 560ca41044a2d6..a3b10cfa5e1666 100644 --- a/package.json +++ b/package.json @@ -422,6 +422,7 @@ "@kbn/es-ui-shared-plugin": "link:src/plugins/es_ui_shared", "@kbn/eso-model-version-example": "link:examples/eso_model_version_example", "@kbn/eso-plugin": "link:x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin", + "@kbn/esql-utils": "link:packages/kbn-esql-utils", "@kbn/event-annotation-common": "link:packages/kbn-event-annotation-common", "@kbn/event-annotation-components": "link:packages/kbn-event-annotation-components", "@kbn/event-annotation-listing-plugin": "link:src/plugins/event_annotation_listing", diff --git a/packages/kbn-es-query/tsconfig.json b/packages/kbn-es-query/tsconfig.json index 9dda3fcbcc2cf3..e7bd8b3c037d5c 100644 --- a/packages/kbn-es-query/tsconfig.json +++ b/packages/kbn-es-query/tsconfig.json @@ -15,7 +15,7 @@ "kbn_references": [ "@kbn/utility-types", "@kbn/i18n", - "@kbn/safer-lodash-set", + "@kbn/safer-lodash-set" ], "exclude": [ "target/**/*", diff --git a/packages/kbn-esql-utils/README.md b/packages/kbn-esql-utils/README.md new file mode 100644 index 00000000000000..694c90b3416bd5 --- /dev/null +++ b/packages/kbn-esql-utils/README.md @@ -0,0 +1,4 @@ +# @kbn/esql-utils + +This package contains utilities for ES|QL. + diff --git a/packages/kbn-esql-utils/index.ts b/packages/kbn-esql-utils/index.ts new file mode 100644 index 00000000000000..e478dbbf32d95b --- /dev/null +++ b/packages/kbn-esql-utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { getESQLAdHocDataview } from './src'; diff --git a/packages/kbn-esql-utils/jest.config.js b/packages/kbn-esql-utils/jest.config.js new file mode 100644 index 00000000000000..32340a902d76b5 --- /dev/null +++ b/packages/kbn-esql-utils/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-esql-utils'], +}; diff --git a/packages/kbn-esql-utils/kibana.jsonc b/packages/kbn-esql-utils/kibana.jsonc new file mode 100644 index 00000000000000..4dd00764681a5d --- /dev/null +++ b/packages/kbn-esql-utils/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/esql-utils", + "owner": "@elastic/kibana-visualizations" +} diff --git a/packages/kbn-esql-utils/package.json b/packages/kbn-esql-utils/package.json new file mode 100644 index 00000000000000..57425f11e94e78 --- /dev/null +++ b/packages/kbn-esql-utils/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/esql-utils", + "private": true, + "version": "1.0.0", + "license": "SSPL-1.0 OR Elastic License 2.0", + "sideEffects": false +} \ No newline at end of file diff --git a/packages/kbn-esql-utils/src/index.ts b/packages/kbn-esql-utils/src/index.ts new file mode 100644 index 00000000000000..a50ff5c59e7986 --- /dev/null +++ b/packages/kbn-esql-utils/src/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './utils'; diff --git a/packages/kbn-esql-utils/src/utils/index.ts b/packages/kbn-esql-utils/src/utils/index.ts new file mode 100644 index 00000000000000..7ac96e272fcf7a --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/index.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; + +// uses browser sha256 method with fallback if unavailable +async function sha256(str: string) { + if (crypto.subtle) { + const enc = new TextEncoder(); + const hash = await crypto.subtle.digest('SHA-256', enc.encode(str)); + return Array.from(new Uint8Array(hash)) + .map((v) => v.toString(16).padStart(2, '0')) + .join(''); + } else { + const { sha256: sha256fn } = await import('./sha256'); + return sha256fn(str); + } +} + +// Some applications need to have a dataview to work properly with ES|QL queries +// This is a helper to create one. The id is constructed from the indexpattern. +// As there are no runtime fields or field formatters or default time fields +// the same adhoc dataview can be constructed/used. This comes with great advantages such +// as solving the problem descibed here https://github.com/elastic/kibana/issues/168131 +export async function getESQLAdHocDataview( + indexPattern: string, + dataViewsService: DataViewsPublicPluginStart +) { + return await dataViewsService.create({ + title: indexPattern, + id: await sha256(`esql-${indexPattern}`), + }); +} diff --git a/packages/kbn-esql-utils/src/utils/sha256.ts b/packages/kbn-esql-utils/src/utils/sha256.ts new file mode 100644 index 00000000000000..dea53f74eeb562 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/sha256.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Sha256 } from '@kbn/crypto-browser'; + +export const sha256 = async (str: string) => new Sha256().update(str).digest('hex'); diff --git a/packages/kbn-esql-utils/tsconfig.json b/packages/kbn-esql-utils/tsconfig.json new file mode 100644 index 00000000000000..2fe775fb7d586f --- /dev/null +++ b/packages/kbn-esql-utils/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/data-views-plugin", + "@kbn/crypto-browser", + ] +} diff --git a/src/plugins/discover/public/application/main/utils/get_data_view_by_text_based_query_lang.ts b/src/plugins/discover/public/application/main/utils/get_data_view_by_text_based_query_lang.ts index 09ac5f1e876861..257fa050ba83c9 100644 --- a/src/plugins/discover/public/application/main/utils/get_data_view_by_text_based_query_lang.ts +++ b/src/plugins/discover/public/application/main/utils/get_data_view_by_text_based_query_lang.ts @@ -10,6 +10,7 @@ import { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery, } from '@kbn/es-query'; +import { getESQLAdHocDataview } from '@kbn/esql-utils'; import { DataView } from '@kbn/data-views-plugin/common'; import { DiscoverServices } from '../../../build_services'; @@ -32,9 +33,7 @@ export async function getDataViewByTextBasedQueryLang( currentDataView?.isPersisted() || indexPatternFromQuery !== currentDataView?.getIndexPattern() ) { - const dataViewObj = await services.dataViews.create({ - title: indexPatternFromQuery, - }); + const dataViewObj = await getESQLAdHocDataview(indexPatternFromQuery, services.dataViews); if (dataViewObj.fields.getByName('@timestamp')?.type === 'date') { dataViewObj.timeFieldName = '@timestamp'; diff --git a/src/plugins/discover/tsconfig.json b/src/plugins/discover/tsconfig.json index def0cb5c85c021..bbb9d7495b2b67 100644 --- a/src/plugins/discover/tsconfig.json +++ b/src/plugins/discover/tsconfig.json @@ -81,6 +81,7 @@ "@kbn/shared-ux-button-toolbar", "@kbn/serverless", "@kbn/deeplinks-observability", + "@kbn/esql-utils", "@kbn/managed-content-badge" ], "exclude": ["target/**/*"] diff --git a/tsconfig.base.json b/tsconfig.base.json index 7a354cc514608e..e2518c13bbce6c 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -760,6 +760,8 @@ "@kbn/eso-model-version-example/*": ["examples/eso_model_version_example/*"], "@kbn/eso-plugin": ["x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin"], "@kbn/eso-plugin/*": ["x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin/*"], + "@kbn/esql-utils": ["packages/kbn-esql-utils"], + "@kbn/esql-utils/*": ["packages/kbn-esql-utils/*"], "@kbn/event-annotation-common": ["packages/kbn-event-annotation-common"], "@kbn/event-annotation-common/*": ["packages/kbn-event-annotation-common/*"], "@kbn/event-annotation-components": ["packages/kbn-event-annotation-components"], diff --git a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts index 4555f3f8a576d2..ada2afe14810e6 100644 --- a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts +++ b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts @@ -7,6 +7,7 @@ import { i18n } from '@kbn/i18n'; import { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery } from '@kbn/es-query'; import type { AggregateQuery, Query, Filter } from '@kbn/es-query'; +import { getESQLAdHocDataview } from '@kbn/esql-utils'; import { fetchFieldsFromESQL } from '@kbn/text-based-editor'; import type { DataView, DataViewSpec } from '@kbn/data-views-plugin/public'; import type { Suggestion } from '../../../types'; @@ -48,11 +49,10 @@ export const getSuggestions = async ( return adHoc.name === indexPattern; }); - const dataView = await deps.dataViews.create( - dataViewSpec ?? { - title: indexPattern, - } - ); + const dataView = dataViewSpec + ? await deps.dataViews.create(dataViewSpec) + : await getESQLAdHocDataview(indexPattern, deps.dataViews); + if (dataView.fields.getByName('@timestamp')?.type === 'date' && !dataViewSpec) { dataView.timeFieldName = '@timestamp'; } diff --git a/x-pack/plugins/lens/public/datasources/text_based/utils.ts b/x-pack/plugins/lens/public/datasources/text_based/utils.ts index 856e608d347e1b..708c499d59908b 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/utils.ts +++ b/x-pack/plugins/lens/public/datasources/text_based/utils.ts @@ -7,7 +7,7 @@ import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { ExpressionsStart } from '@kbn/expressions-plugin/public'; - +import { getESQLAdHocDataview } from '@kbn/esql-utils'; import { type AggregateQuery, getIndexPatternFromSQLQuery, @@ -16,7 +16,6 @@ import { import type { DatatableColumn } from '@kbn/expressions-plugin/public'; import { generateId } from '../../id_generator'; import { fetchDataFromAggregateQuery } from './fetch_data_from_aggregate_query'; - import type { IndexPatternRef, TextBasedPrivateState, TextBasedLayerColumn } from './types'; import type { DataViewsState } from '../../state_management'; import { addColumnsToCache } from './fieldlist_cache'; @@ -89,9 +88,8 @@ export async function getStateFromAggregateQuery( let columnsFromQuery: DatatableColumn[] = []; let timeFieldName; try { - const dataView = await dataViews.create({ - title: indexPattern, - }); + const dataView = await getESQLAdHocDataview(indexPattern, dataViews); + if (dataView && dataView.id) { if (dataView?.fields?.getByName('@timestamp')?.type === 'date') { dataView.timeFieldName = '@timestamp'; diff --git a/x-pack/plugins/lens/tsconfig.json b/x-pack/plugins/lens/tsconfig.json index 01470e01785f61..779aa6886b05e6 100644 --- a/x-pack/plugins/lens/tsconfig.json +++ b/x-pack/plugins/lens/tsconfig.json @@ -105,6 +105,7 @@ "@kbn/visualization-utils", "@kbn/test-eui-helpers", "@kbn/shared-ux-utility", + "@kbn/esql-utils", "@kbn/text-based-editor", "@kbn/managed-content-badge", "@kbn/sort-predicates", diff --git a/yarn.lock b/yarn.lock index c2634b0431587f..754fe46ae1ea73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4584,6 +4584,10 @@ version "0.0.0" uid "" +"@kbn/esql-utils@link:packages/kbn-esql-utils": + version "0.0.0" + uid "" + "@kbn/event-annotation-common@link:packages/kbn-event-annotation-common": version "0.0.0" uid "" From 7abab622b8bfe2f87904dc8035d25a5d101d3589 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 18:36:50 +0000 Subject: [PATCH 11/20] skip flaky suite (#175934) --- .../plugins/cases/public/components/create/severity.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cases/public/components/create/severity.test.tsx b/x-pack/plugins/cases/public/components/create/severity.test.tsx index bf81dfc357fc73..91f51578f8cea1 100644 --- a/x-pack/plugins/cases/public/components/create/severity.test.tsx +++ b/x-pack/plugins/cases/public/components/create/severity.test.tsx @@ -31,7 +31,8 @@ const MockHookWrapperComponent: React.FC = ({ children }) => { return
{children}
; }; -describe('Severity form field', () => { +// FLAKY: https://github.com/elastic/kibana/issues/175934 +describe.skip('Severity form field', () => { let appMockRender: AppMockRenderer; beforeEach(() => { appMockRender = createAppMockRenderer(); From 2ba824b8893fd57d47f0dd44227a9046cd0597f3 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 18:37:11 +0000 Subject: [PATCH 12/20] skip flaky suite (#175935) --- x-pack/plugins/cases/public/components/create/severity.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/cases/public/components/create/severity.test.tsx b/x-pack/plugins/cases/public/components/create/severity.test.tsx index 91f51578f8cea1..5431d17d5f54c1 100644 --- a/x-pack/plugins/cases/public/components/create/severity.test.tsx +++ b/x-pack/plugins/cases/public/components/create/severity.test.tsx @@ -32,6 +32,7 @@ const MockHookWrapperComponent: React.FC = ({ children }) => { return
{children}
; }; // FLAKY: https://github.com/elastic/kibana/issues/175934 +// FLAKY: https://github.com/elastic/kibana/issues/175935 describe.skip('Severity form field', () => { let appMockRender: AppMockRenderer; beforeEach(() => { From 53c3907529f6cf15c518f5d6aa74ecfbb998e524 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:52:31 -0600 Subject: [PATCH 13/20] [ML] Add support for ES|QL in Data visualizer (#174188) ## Summary This PR adds support for ES|QL queries in Data visualizer. Screenshot 2024-01-26 at 17 07 59 Screenshot 2024-01-26 at 17 12 39 https://github.com/elastic/kibana/assets/43350163/a3f540e9-461d-4ebc-bd69-de4ffa2bc554 ### Changes: - Add a new card from the Data visualizer main page - Add a link from the ML navigation Screenshot 2024-01-08 at 18 03 50 - Added a new button to Use ES|QL Screenshot 2024-01-09 at 11 23 09 - Support for **keyword**, **text**, **numeric**, **boolean**, **date**, and **ip** fields Screenshot 2024-01-09 at 11 24 38 Screenshot 2024-01-09 at 11 25 25 Screenshot 2024-01-09 at 11 44 02 Screenshot 2024-01-10 at 12 01 42 - Default to user's fieldFormats for fields that are dynamic generated by ES|QL, else use Data view's format - Default to Data view's setting (e.g. type `bytes` in this case for field `bytes_normal_counter`) Screenshot 2024-01-10 at 12 10 38 - Default to user's fieldFormats formatting for dynamically generated fields (e.g. type `number` in this case for field `avg_price`) Screenshot 2024-01-10 at 12 01 03 - Add a new UI control to allow users to limit analysis to 5,000 - 10,000 - 100,000 - 1,000,000, rows. This speeds up fetching of the stats for big data sets and avoid potential circuit breaking exceptions. - Break overall stats request into smaller parallel requests (which prevent time out or payload too big due by too many fields), at 10 requests at a time - Break field stats for individual fields into more efficient batches (which prevent time out or payload too big due by too many fields), at 10 requests at at ime - Improve error handling by propagating up the error AND the ES|QL request in both the UI and the developer's console (for better debugging) - Improve error handling in field stats rows: If one field, or a group of fields, say 'keyword' fields fail to fetch for some reasons, it will show error for that field but not affect all other fields. Screenshot 2024-01-26 at 16 04 28 - Add deep linking in the top search bar Screenshot 2024-01-26 at 16 56 49 - More robust support for keyword fields with geo data Screenshot 2024-01-26 at 16 55 01 ### Todos: - [x] Add earliest/latest for date time fields -> Current blocker: escape special characters in esql variable names - [x] Fix formatting of numbers for dynamic query, where we don't know the formatting based on the data view - [x] Fix date time 'Update' not updating until Refresh is clicked - [x] Better optimization to not fetch distribution & expanded row content for pages that are not visible ### Good to have: - [ ] Investigate bringing back the +/- filter buttons (either by modifying the ES|QL query directly or by adding separate DSL filters?) ------------ ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + package.json | 1 + packages/deeplinks/ml/deep_links.ts | 1 + packages/default-nav/ml/default_navigation.ts | 9 + tsconfig.base.json | 2 + .../packages/ml/cancellable_search/README.md | 3 + .../packages/ml/cancellable_search/index.ts | 8 + .../ml/cancellable_search/jest.config.js | 12 + .../ml/cancellable_search/kibana.jsonc | 5 + .../ml/cancellable_search/package.json | 8 + .../src/use_cancellable_search.ts | 73 ++ .../ml/cancellable_search/tsconfig.json | 21 + .../src/components/date_picker_wrapper.tsx | 14 +- .../common/types/field_stats.ts | 13 +- x-pack/plugins/data_visualizer/kibana.jsonc | 3 +- .../document_count_content.tsx | 142 +-- .../examples_list/examples_list.tsx | 2 +- .../expanded_row/index_based_expanded_row.tsx | 5 +- .../field_data_row/document_stats.tsx | 3 +- .../field_data_row/top_values_preview.tsx | 2 +- .../data_visualizer_stats_table.tsx | 5 +- .../components/top_values/top_values.tsx | 4 +- .../public/application/common/constants.ts | 2 + .../common/util/promise_all_settled_utils.ts | 13 + .../data_drift/use_data_drift_result.ts | 7 +- .../index_data_visualizer_esql.tsx | 815 ++++++++++++++++++ .../index_data_visualizer_view.tsx | 5 +- .../search_panel/esql/limit_size.tsx | 83 ++ .../hooks/esql/use_esql_field_stats_data.ts | 170 ++++ .../hooks/esql/use_esql_overall_stats_data.ts | 395 +++++++++ .../hooks/use_data_visualizer_grid_data.ts | 12 +- .../hooks/use_field_stats.ts | 2 +- .../hooks/use_overall_stats.ts | 1 + .../index_data_visualizer.tsx | 31 +- .../esql_requests/get_boolean_field_stats.ts | 111 +++ .../get_count_and_cardinality.ts | 200 +++++ .../esql_requests/get_date_field_stats.ts | 75 ++ .../esql_requests/get_keyword_fields.ts | 99 +++ .../esql_requests/get_numeric_field_stats.ts | 149 ++++ .../esql_requests/get_text_field_stats.ts | 66 ++ .../esql_requests/handle_error.ts | 40 + .../requests/esql_utils.test.ts | 19 + .../search_strategy/requests/esql_utils.ts | 42 + .../get_data_view_by_index_pattern.ts | 47 + .../requests/get_document_stats.ts | 3 +- .../requests/get_numeric_field_stats.ts | 6 +- .../requests/get_string_field_stats.ts | 5 - .../types/esql_data_visualizer.ts | 19 + .../types/index_data_visualizer_state.ts | 1 - .../index_data_visualizer/types/storage.ts | 5 + .../utils/get_supported_aggs.ts | 8 + .../utils/process_distribution_data.ts | 3 + x-pack/plugins/data_visualizer/tsconfig.json | 3 + x-pack/plugins/ml/common/constants/locator.ts | 1 + x-pack/plugins/ml/common/types/locator.ts | 1 + .../components/ml_page/side_nav.tsx | 10 + .../datavisualizer_selector.tsx | 52 ++ .../index_based/index_data_visualizer.tsx | 27 +- .../new_job/pages/index_or_search/page.tsx | 18 +- .../components/navigate_to_page_button.tsx | 25 + .../routes/datavisualizer/index_based.tsx | 27 +- .../routes/new_job/index_or_search.tsx | 37 +- .../plugins/ml/public/locator/ml_locator.ts | 1 + .../search_deep_links.ts | 11 + yarn.lock | 4 + 65 files changed, 2850 insertions(+), 138 deletions(-) create mode 100644 x-pack/packages/ml/cancellable_search/README.md create mode 100644 x-pack/packages/ml/cancellable_search/index.ts create mode 100644 x-pack/packages/ml/cancellable_search/jest.config.js create mode 100644 x-pack/packages/ml/cancellable_search/kibana.jsonc create mode 100644 x-pack/packages/ml/cancellable_search/package.json create mode 100644 x-pack/packages/ml/cancellable_search/src/use_cancellable_search.ts create mode 100644 x-pack/packages/ml/cancellable_search/tsconfig.json create mode 100644 x-pack/plugins/data_visualizer/public/application/common/util/promise_all_settled_utils.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_field_stats_data.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/handle_error.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.test.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_data_view_by_index_pattern.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/esql_data_visualizer.ts create mode 100644 x-pack/plugins/ml/public/application/routing/components/navigate_to_page_button.tsx diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b120feb392a1c9..b07f76b9683f63 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -530,6 +530,7 @@ x-pack/packages/maps/vector_tile_utils @elastic/kibana-gis x-pack/plugins/metrics_data_access @elastic/obs-knowledge-team x-pack/packages/ml/agg_utils @elastic/ml-ui x-pack/packages/ml/anomaly_utils @elastic/ml-ui +x-pack/packages/ml/cancellable_search @elastic/ml-ui x-pack/packages/ml/category_validator @elastic/ml-ui x-pack/packages/ml/chi2test @elastic/ml-ui x-pack/packages/ml/creation_wizard_utils @elastic/ml-ui diff --git a/package.json b/package.json index a3b10cfa5e1666..79ab0211e902f6 100644 --- a/package.json +++ b/package.json @@ -550,6 +550,7 @@ "@kbn/metrics-data-access-plugin": "link:x-pack/plugins/metrics_data_access", "@kbn/ml-agg-utils": "link:x-pack/packages/ml/agg_utils", "@kbn/ml-anomaly-utils": "link:x-pack/packages/ml/anomaly_utils", + "@kbn/ml-cancellable-search": "link:x-pack/packages/ml/cancellable_search", "@kbn/ml-category-validator": "link:x-pack/packages/ml/category_validator", "@kbn/ml-chi2test": "link:x-pack/packages/ml/chi2test", "@kbn/ml-creation-wizard-utils": "link:x-pack/packages/ml/creation_wizard_utils", diff --git a/packages/deeplinks/ml/deep_links.ts b/packages/deeplinks/ml/deep_links.ts index 1c16543512b095..49b644c1ead642 100644 --- a/packages/deeplinks/ml/deep_links.ts +++ b/packages/deeplinks/ml/deep_links.ts @@ -27,6 +27,7 @@ export type LinkId = | 'nodesOverview' | 'nodes' | 'memoryUsage' + | 'esqlDataVisualizer' | 'dataVisualizer' | 'fileUpload' | 'indexDataVisualizer' diff --git a/packages/default-nav/ml/default_navigation.ts b/packages/default-nav/ml/default_navigation.ts index f82ec721b22eeb..cc00154907e73e 100644 --- a/packages/default-nav/ml/default_navigation.ts +++ b/packages/default-nav/ml/default_navigation.ts @@ -121,6 +121,15 @@ export const defaultNavigation: MlNodeDefinition = { ); }, }, + { + title: i18n.translate('defaultNavigation.ml.esqlDataVisualizer', { + defaultMessage: 'ES|QL', + }), + link: 'ml:esqlDataVisualizer', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.includes(prepend('/app/ml/datavisualizer/esql')); + }, + }, { title: i18n.translate('defaultNavigation.ml.dataComparison', { defaultMessage: 'Data drift', diff --git a/tsconfig.base.json b/tsconfig.base.json index e2518c13bbce6c..9e7531586d5daf 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1054,6 +1054,8 @@ "@kbn/ml-agg-utils/*": ["x-pack/packages/ml/agg_utils/*"], "@kbn/ml-anomaly-utils": ["x-pack/packages/ml/anomaly_utils"], "@kbn/ml-anomaly-utils/*": ["x-pack/packages/ml/anomaly_utils/*"], + "@kbn/ml-cancellable-search": ["x-pack/packages/ml/cancellable_search"], + "@kbn/ml-cancellable-search/*": ["x-pack/packages/ml/cancellable_search/*"], "@kbn/ml-category-validator": ["x-pack/packages/ml/category_validator"], "@kbn/ml-category-validator/*": ["x-pack/packages/ml/category_validator/*"], "@kbn/ml-chi2test": ["x-pack/packages/ml/chi2test"], diff --git a/x-pack/packages/ml/cancellable_search/README.md b/x-pack/packages/ml/cancellable_search/README.md new file mode 100644 index 00000000000000..3f7463e69af9e2 --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/README.md @@ -0,0 +1,3 @@ +# @kbn/ml-cancellable-search + +React hook for cancellable data searching diff --git a/x-pack/packages/ml/cancellable_search/index.ts b/x-pack/packages/ml/cancellable_search/index.ts new file mode 100644 index 00000000000000..7ae3ce1a71ef58 --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { useCancellableSearch, type UseCancellableSearch } from './src/use_cancellable_search'; diff --git a/x-pack/packages/ml/cancellable_search/jest.config.js b/x-pack/packages/ml/cancellable_search/jest.config.js new file mode 100644 index 00000000000000..450cf6662aa45d --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/packages/ml/cancellable_search'], +}; diff --git a/x-pack/packages/ml/cancellable_search/kibana.jsonc b/x-pack/packages/ml/cancellable_search/kibana.jsonc new file mode 100644 index 00000000000000..2006bfd7467110 --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/ml-cancellable-search", + "owner": "@elastic/ml-ui" +} diff --git a/x-pack/packages/ml/cancellable_search/package.json b/x-pack/packages/ml/cancellable_search/package.json new file mode 100644 index 00000000000000..79630662bc5c33 --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/ml-cancellable-search", + "description": "React hook for cancellable data searching", + "author": "Machine Learning UI", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/packages/ml/cancellable_search/src/use_cancellable_search.ts b/x-pack/packages/ml/cancellable_search/src/use_cancellable_search.ts new file mode 100644 index 00000000000000..78e7e4a17b1fae --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/src/use_cancellable_search.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useRef, useState } from 'react'; +import { type IKibanaSearchResponse, isRunningResponse } from '@kbn/data-plugin/common'; +import { tap } from 'rxjs/operators'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; + +export interface UseCancellableSearch { + runRequest: ( + requestBody: RequestBody, + options?: object + ) => Promise; + cancelRequest: () => void; + isLoading: boolean; +} + +// Similar to aiops/hooks/use_cancellable_search.ts +export function useCancellableSearch(data: DataPublicPluginStart) { + const abortController = useRef(new AbortController()); + const [isLoading, setIsFetching] = useState(false); + + const runRequest = useCallback( + ( + requestBody: RequestBody, + options = {} + ): Promise => { + return new Promise((resolve, reject) => { + data.search + .search(requestBody, { + abortSignal: abortController.current.signal, + ...options, + }) + .pipe( + tap(() => { + setIsFetching(true); + }) + ) + .subscribe({ + next: (result) => { + if (!isRunningResponse(result)) { + setIsFetching(false); + resolve(result); + } else { + // partial results + // Ignore partial results for now. + // An issue with the search function means partial results are not being returned correctly. + } + }, + error: (error) => { + if (error.name === 'AbortError') { + return resolve(null); + } + setIsFetching(false); + reject(error); + }, + }); + }); + }, + [data.search] + ); + + const cancelRequest = useCallback(() => { + abortController.current.abort(); + abortController.current = new AbortController(); + }, []); + + return { runRequest, cancelRequest, isLoading }; +} diff --git a/x-pack/packages/ml/cancellable_search/tsconfig.json b/x-pack/packages/ml/cancellable_search/tsconfig.json new file mode 100644 index 00000000000000..733865038a43f9 --- /dev/null +++ b/x-pack/packages/ml/cancellable_search/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*", + ], + "kbn_references": [ + "@kbn/data-plugin" + ] +} diff --git a/x-pack/packages/ml/date_picker/src/components/date_picker_wrapper.tsx b/x-pack/packages/ml/date_picker/src/components/date_picker_wrapper.tsx index 8d500f7d1a815c..d16f74561984c1 100644 --- a/x-pack/packages/ml/date_picker/src/components/date_picker_wrapper.tsx +++ b/x-pack/packages/ml/date_picker/src/components/date_picker_wrapper.tsx @@ -90,6 +90,10 @@ interface DatePickerWrapperProps { * Boolean flag to set use of flex group wrapper */ flexGroup?: boolean; + /** + * Boolean flag to disable the date picker + */ + isDisabled?: boolean; } /** @@ -100,7 +104,14 @@ interface DatePickerWrapperProps { * @returns {React.ReactElement} The DatePickerWrapper component. */ export const DatePickerWrapper: FC = (props) => { - const { isAutoRefreshOnly, isLoading = false, showRefresh, width, flexGroup = true } = props; + const { + isAutoRefreshOnly, + isLoading = false, + showRefresh, + width, + flexGroup = true, + isDisabled = false, + } = props; const { data, notifications: { toasts }, @@ -292,6 +303,7 @@ export const DatePickerWrapper: FC = (props) => { commonlyUsedRanges={commonlyUsedRanges} updateButtonProps={{ iconOnly: isWithinLBreakpoint, fill: false }} width={width} + isDisabled={isDisabled} /> {showRefresh === true || !isTimeRangeSelectorEnabled ? ( diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index 5fef7b4ae8f778..ea6956479bfec5 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -72,18 +72,21 @@ export const isIKibanaSearchResponse = (arg: unknown): arg is IKibanaSearchRespo return isPopulatedObject(arg, ['rawResponse']); }; -export interface NumericFieldStats { +export interface NonSampledNumericFieldStats { fieldName: string; count?: number; min?: number; max?: number; avg?: number; + median?: number; + distribution?: Distribution; +} + +export interface NumericFieldStats extends NonSampledNumericFieldStats { isTopValuesSampled: boolean; topValues: Bucket[]; topValuesSampleSize: number; topValuesSamplerShardSize: number; - median?: number; - distribution?: Distribution; } export interface StringFieldStats { @@ -178,6 +181,7 @@ export type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData; export type BatchStats = + | NonSampledNumericFieldStats | NumericFieldStats | StringFieldStats | BooleanFieldStats @@ -186,6 +190,7 @@ export type BatchStats = | FieldExamples; export type FieldStats = + | NonSampledNumericFieldStats | NumericFieldStats | StringFieldStats | BooleanFieldStats @@ -199,7 +204,6 @@ export function isValidFieldStats(arg: unknown): arg is FieldStats { export interface FieldStatsCommonRequestParams { index: string; - samplerShardSize: number; timeFieldName?: string; earliestMs?: number | undefined; latestMs?: number | undefined; @@ -222,7 +226,6 @@ export interface OverallStatsSearchStrategyParams { aggInterval: TimeBucketsInterval; intervalMs?: number; searchQuery: Query['query']; - samplerShardSize: number; index: string; timeFieldName?: string; runtimeFieldMap?: estypes.MappingRuntimeFields; diff --git a/x-pack/plugins/data_visualizer/kibana.jsonc b/x-pack/plugins/data_visualizer/kibana.jsonc index 79a1e1fedacaf0..e3c6e8514dabdf 100644 --- a/x-pack/plugins/data_visualizer/kibana.jsonc +++ b/x-pack/plugins/data_visualizer/kibana.jsonc @@ -36,7 +36,8 @@ "esUiShared", "fieldFormats", "uiActions", - "lens" + "lens", + "textBasedLanguages", ] } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index 3d1268140df170..6403e2c1e81305 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -38,8 +38,9 @@ export interface Props { samplingProbability?: number | null; setSamplingProbability?: (value: number | null) => void; randomSamplerPreference?: RandomSamplerOption; - setRandomSamplerPreference: (value: RandomSamplerOption) => void; + setRandomSamplerPreference?: (value: RandomSamplerOption) => void; loading: boolean; + showSettings?: boolean; } const CalculatingProbabilityMessage = ( @@ -61,6 +62,7 @@ export const DocumentCountContent: FC = ({ loading, randomSamplerPreference, setRandomSamplerPreference, + showSettings = true, }) => { const [showSamplingOptionsPopover, setShowSamplingOptionsPopover] = useState(false); @@ -120,75 +122,79 @@ export const DocumentCountContent: FC = ({ <> - - - + - - } - isOpen={showSamplingOptionsPopover} - closePopover={closeSamplingOptions} - panelPaddingSize="none" - anchorPosition="downLeft" - > - - - - - - - - - setRandomSamplerPreference(e.target.value as RandomSamplerOption) - } - /> - - - {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_MANUAL ? ( - - ) : null} - - {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_AUTOMATIC ? ( - loading ? ( - CalculatingProbabilityMessage - ) : ( - - ) - ) : null} - - - - + > + + + } + isOpen={showSamplingOptionsPopover} + closePopover={closeSamplingOptions} + panelPaddingSize="none" + anchorPosition="downLeft" + > + + + + + + + {setRandomSamplerPreference ? ( + + + setRandomSamplerPreference(e.target.value as RandomSamplerOption) + } + /> + + ) : null} + + {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_MANUAL ? ( + + ) : null} + + {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_AUTOMATIC ? ( + loading ? ( + CalculatingProbabilityMessage + ) : ( + + ) + ) : null} + + + + + ) : null} ; } -const EMPTY_EXAMPLE = i18n.translate( +export const EMPTY_EXAMPLE = i18n.translate( 'xpack.dataVisualizer.dataGrid.field.examplesList.emptyExampleMessage', { defaultMessage: '(empty)' } ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx index e164a37307daf6..f921d66e2cd1e8 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx @@ -31,18 +31,21 @@ export const IndexBasedDataVisualizerExpandedRow = ({ combinedQuery, onAddFilter, totalDocuments, + typeAccessor = 'type', }: { item: FieldVisConfig; dataView: DataView | undefined; combinedQuery: CombinedQuery; totalDocuments?: number; + typeAccessor?: 'type' | 'secondaryType'; /** * Callback to add a filter to filter bar */ onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; }) => { const config = { ...item, stats: { ...item.stats, totalDocuments } }; - const { loading, type, existsInDocs, fieldName } = config; + const { loading, existsInDocs, fieldName } = config; + const type = config[typeAccessor]; const dvExpandedRow = useExpandedRowCss(); function getCardContent() { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 79a878fb117345..2a7ca2fedc0bd4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -31,6 +31,7 @@ export const DocumentStat = ({ config, showIcon, totalCount }: Props) => { if (stats === undefined) return null; const { count, sampleCount } = stats; + const total = sampleCount ?? totalCount; // If field exists is docs but we don't have count stats then don't show @@ -39,7 +40,7 @@ export const DocumentStat = ({ config, showIcon, totalCount }: Props) => { count ?? (isIndexBasedFieldVisConfig(config) && config.existsInDocs === true ? undefined : 0); const docsPercent = valueCount !== undefined && total !== undefined - ? `(${roundToDecimalPlace((valueCount / total) * 100)}%)` + ? `(${total === 0 ? 0 : roundToDecimalPlace((valueCount / total) * 100)}%)` : null; const content = ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/top_values_preview.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/top_values_preview.tsx index 9ff7b12364aaa9..f39a45f56c2ba2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/top_values_preview.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/top_values_preview.tsx @@ -24,7 +24,7 @@ export const TopValuesPreview: FC = ({ config, isNumeric const data: OrdinalDataItem[] = topValues.map((d) => ({ ...d, - key: d.key.toString(), + key: d.key?.toString(), })); const chartData: ChartData = { cardinality, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 71da03d5938bb8..b0a5ae1d8a47df 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -341,7 +341,10 @@ export const DataVisualizerTable = ({ return ; } - if (item.type === SUPPORTED_FIELD_TYPES.NUMBER) { + if ( + item.type === SUPPORTED_FIELD_TYPES.NUMBER || + item.secondaryType === SUPPORTED_FIELD_TYPES.NUMBER + ) { if (isIndexBasedFieldVisConfig(item) && item.stats?.distribution !== undefined) { // If the cardinality is only low, show the top values instead of a distribution chart return item.stats?.distribution?.percentiles.length <= 2 ? ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 0b2475789091f6..111b0a2113403b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -28,6 +28,7 @@ import { kibanaFieldFormat } from '../utils'; import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header'; import { FieldVisStats } from '../../../../../common/types'; import { ExpandedRowPanel } from '../stats_table/components/field_data_expanded_row/expanded_row_panel'; +import { EMPTY_EXAMPLE } from '../examples_list/examples_list'; interface Props { stats: FieldVisStats | undefined; @@ -115,7 +116,8 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, > {Array.isArray(topValues) ? topValues.map((value) => { - const fieldValue = value.key_as_string ?? value.key.toString(); + const fieldValue = + value.key_as_string ?? (value.key ? value.key.toString() : EMPTY_EXAMPLE); return ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/constants.ts b/x-pack/plugins/data_visualizer/public/application/common/constants.ts index 3c2ab184c54c10..9d5f5673e94357 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/constants.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/constants.ts @@ -7,6 +7,8 @@ import { i18n } from '@kbn/i18n'; +export const DEFAULT_BAR_TARGET = 75; + export const INDEX_DATA_VISUALIZER_NAME = i18n.translate( 'xpack.dataVisualizer.chrome.help.appName', { diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/promise_all_settled_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/util/promise_all_settled_utils.ts new file mode 100644 index 00000000000000..ade4b5ac04a043 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/util/promise_all_settled_utils.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const isFulfilled = ( + input: PromiseSettledResult> +): input is PromiseFulfilledResult> => input.status === 'fulfilled'; +export const isRejected = ( + input: PromiseSettledResult> +): input is PromiseRejectedResult => input.status === 'rejected'; diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts index 88b597de5c1283..07b74677e8ea9d 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts @@ -54,6 +54,7 @@ import { TimeRange, ComparisonHistogram, } from './types'; +import { isFulfilled, isRejected } from '../common/util/promise_all_settled_utils'; export const getDataComparisonType = (kibanaType: string): DataDriftField['type'] => { switch (kibanaType) { @@ -588,12 +589,6 @@ const fetchHistogramData = async ({ } }; -const isFulfilled = ( - input: PromiseSettledResult> -): input is PromiseFulfilledResult> => input.status === 'fulfilled'; -const isRejected = (input: PromiseSettledResult>): input is PromiseRejectedResult => - input.status === 'rejected'; - type EsRequestParams = NonNullable< IKibanaSearchRequest>['params'] >; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx new file mode 100644 index 00000000000000..2edd03e7f27af6 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx @@ -0,0 +1,815 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable react-hooks/exhaustive-deps */ + +import { css } from '@emotion/react'; +import React, { FC, useEffect, useMemo, useState, useCallback, useRef } from 'react'; +import type { Required } from 'utility-types'; +import { + FullTimeRangeSelector, + mlTimefilterRefresh$, + useTimefilter, + DatePickerWrapper, +} from '@kbn/ml-date-picker'; +import { TextBasedLangEditor } from '@kbn/text-based-languages/public'; +import type { AggregateQuery } from '@kbn/es-query'; +import { merge } from 'rxjs'; +import { Comparators } from '@elastic/eui'; + +import { + useEuiBreakpoint, + useIsWithinMaxBreakpoint, + EuiFlexGroup, + EuiFlexItem, + EuiPageTemplate, + EuiPanel, + EuiProgress, + EuiSpacer, +} from '@elastic/eui'; +import { usePageUrlState, useUrlState } from '@kbn/ml-url-state'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; +import { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery } from '@kbn/es-query'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { KBN_FIELD_TYPES } from '@kbn/field-types'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { getFieldType } from '@kbn/field-utils'; +import { UI_SETTINGS } from '@kbn/data-service'; +import type { SupportedFieldType } from '../../../../../common/types'; +import { useCurrentEuiTheme } from '../../../common/hooks/use_current_eui_theme'; +import type { FieldVisConfig } from '../../../common/components/stats_table/types'; +import { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; +import type { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { GetAdditionalLinks } from '../../../common/components/results_links'; +import { DocumentCountContent } from '../../../common/components/document_count_content'; +import { useTimeBuckets } from '../../../common/hooks/use_time_buckets'; +import { + DataVisualizerTable, + ItemIdToExpandedRowMap, +} from '../../../common/components/stats_table'; +import type { + MetricFieldsStats, + TotalFieldsStats, +} from '../../../common/components/stats_table/components/field_count_stats'; +import { filterFields } from '../../../common/components/fields_stats_grid/filter_fields'; +import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; +import { getOrCreateDataViewByIndexPattern } from '../../search_strategy/requests/get_data_view_by_index_pattern'; +import { FieldCountPanel } from '../../../common/components/field_count_panel'; +import { useESQLFieldStatsData } from '../../hooks/esql/use_esql_field_stats_data'; +import type { NonAggregatableField, OverallStats } from '../../types/overall_stats'; +import { isESQLQuery } from '../../search_strategy/requests/esql_utils'; +import { DEFAULT_BAR_TARGET } from '../../../common/constants'; +import { + type ESQLDefaultLimitSizeOption, + ESQLDefaultLimitSizeSelect, +} from '../search_panel/esql/limit_size'; +import { type Column, useESQLOverallStatsData } from '../../hooks/esql/use_esql_overall_stats_data'; +import { type AggregatableField } from '../../types/esql_data_visualizer'; + +const defaults = getDefaultPageState(); + +interface DataVisualizerPageState { + overallStats: OverallStats; + metricConfigs: FieldVisConfig[]; + totalMetricFieldCount: number; + populatedMetricFieldCount: number; + metricsLoaded: boolean; + nonMetricConfigs: FieldVisConfig[]; + nonMetricsLoaded: boolean; + documentCountStats?: FieldVisConfig; +} + +const defaultSearchQuery = { + match_all: {}, +}; + +export function getDefaultPageState(): DataVisualizerPageState { + return { + overallStats: { + totalCount: 0, + aggregatableExistsFields: [], + aggregatableNotExistsFields: [], + nonAggregatableExistsFields: [], + nonAggregatableNotExistsFields: [], + }, + metricConfigs: [], + totalMetricFieldCount: 0, + populatedMetricFieldCount: 0, + metricsLoaded: false, + nonMetricConfigs: [], + nonMetricsLoaded: false, + documentCountStats: undefined, + }; +} + +interface ESQLDataVisualizerIndexBasedAppState extends DataVisualizerIndexBasedAppState { + limitSize: ESQLDefaultLimitSizeOption; +} + +export interface ESQLDataVisualizerIndexBasedPageUrlState { + pageKey: typeof DATA_VISUALIZER_INDEX_VIEWER; + pageUrlState: Required; +} + +export const getDefaultDataVisualizerListState = ( + overrides?: Partial +): Required => ({ + pageIndex: 0, + pageSize: 25, + sortField: 'fieldName', + sortDirection: 'asc', + visibleFieldTypes: [], + visibleFieldNames: [], + limitSize: '10000', + searchString: '', + searchQuery: defaultSearchQuery, + searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY, + filters: [], + showDistributions: true, + showAllFields: false, + showEmptyFields: false, + probability: null, + rndSamplerPref: 'off', + ...overrides, +}); + +export interface IndexDataVisualizerESQLProps { + getAdditionalLinks?: GetAdditionalLinks; +} + +export const IndexDataVisualizerESQL: FC = (dataVisualizerProps) => { + const { services } = useDataVisualizerKibana(); + const { data, fieldFormats, uiSettings } = services; + const euiTheme = useCurrentEuiTheme(); + + const [query, setQuery] = useState({ esql: '' }); + const [currentDataView, setCurrentDataView] = useState(); + + const updateDataView = (dv: DataView) => { + if (dv.id !== currentDataView?.id) { + setCurrentDataView(dv); + } + }; + const [lastRefresh, setLastRefresh] = useState(0); + + const _timeBuckets = useTimeBuckets(); + const timefilter = useTimefilter({ + timeRangeSelector: true, + autoRefreshSelector: true, + }); + + const indexPattern = useMemo(() => { + let indexPatternFromQuery = ''; + if ('sql' in query) { + indexPatternFromQuery = getIndexPatternFromSQLQuery(query.sql); + } + if ('esql' in query) { + indexPatternFromQuery = getIndexPatternFromESQLQuery(query.esql); + } + // we should find a better way to work with ESQL queries which dont need a dataview + if (indexPatternFromQuery === '') { + return undefined; + } + return indexPatternFromQuery; + }, [query]); + + const restorableDefaults = useMemo( + () => getDefaultDataVisualizerListState({}), + // We just need to load the saved preference when the page is first loaded + + [] + ); + + const [dataVisualizerListState, setDataVisualizerListState] = + usePageUrlState( + DATA_VISUALIZER_INDEX_VIEWER, + restorableDefaults + ); + const [globalState, setGlobalState] = useUrlState('_g'); + + const showEmptyFields = + dataVisualizerListState.showEmptyFields ?? restorableDefaults.showEmptyFields; + const toggleShowEmptyFields = () => { + setDataVisualizerListState({ + ...dataVisualizerListState, + showEmptyFields: !dataVisualizerListState.showEmptyFields, + }); + }; + + const limitSize = dataVisualizerListState.limitSize ?? restorableDefaults.limitSize; + + const updateLimitSize = (newLimitSize: ESQLDefaultLimitSizeOption) => { + setDataVisualizerListState({ + ...dataVisualizerListState, + limitSize: newLimitSize, + }); + }; + + useEffect( + function updateAdhocDataViewFromQuery() { + let unmounted = false; + + const update = async () => { + if (!indexPattern) return; + const dv = await getOrCreateDataViewByIndexPattern( + data.dataViews, + indexPattern, + currentDataView + ); + + if (dv) { + updateDataView(dv); + } + }; + + if (!unmounted) { + update(); + } + + return () => { + unmounted = true; + }; + }, + + [indexPattern, data.dataViews, currentDataView] + ); + + /** Search strategy **/ + const fieldStatsRequest = useMemo(() => { + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = _timeBuckets; + + const tf = timefilter; + + if (!buckets || !tf || (isESQLQuery(query) && query.esql === '')) return; + const activeBounds = tf.getActiveBounds(); + + let earliest: number | undefined; + let latest: number | undefined; + if (activeBounds !== undefined && currentDataView?.timeFieldName !== undefined) { + earliest = activeBounds.min?.valueOf(); + latest = activeBounds.max?.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const barTarget = uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET) ?? DEFAULT_BAR_TARGET; + buckets.setInterval('auto'); + + if (bounds) { + buckets.setBounds(bounds); + buckets.setBarTarget(barTarget); + } + + const aggInterval = buckets.getInterval(); + + const filter = currentDataView?.timeFieldName + ? ({ + bool: { + must: [], + filter: [ + { + range: { + [currentDataView.timeFieldName]: { + format: 'strict_date_optional_time', + gte: timefilter.getTime().from, + lte: timefilter.getTime().to, + }, + }, + }, + ], + should: [], + must_not: [], + }, + } as QueryDslQueryContainer) + : undefined; + return { + earliest, + latest, + aggInterval, + intervalMs: aggInterval?.asMilliseconds(), + searchQuery: query, + limitSize, + sessionId: undefined, + indexPattern, + timeFieldName: currentDataView?.timeFieldName, + runtimeFieldMap: currentDataView?.getRuntimeMappings(), + lastRefresh, + filter, + }; + }, [ + _timeBuckets, + timefilter, + currentDataView?.id, + JSON.stringify(query), + indexPattern, + lastRefresh, + limitSize, + ]); + + useEffect(() => { + // Force refresh on index pattern change + setLastRefresh(Date.now()); + }, [setLastRefresh]); + + useEffect(() => { + if (globalState?.time !== undefined) { + timefilter.setTime({ + from: globalState.time.from, + to: globalState.time.to, + }); + } + }, [JSON.stringify(globalState?.time), timefilter]); + + useEffect(() => { + const timeUpdateSubscription = merge( + timefilter.getTimeUpdate$(), + timefilter.getAutoRefreshFetch$(), + mlTimefilterRefresh$ + ).subscribe(() => { + setGlobalState({ + time: timefilter.getTime(), + refreshInterval: timefilter.getRefreshInterval(), + }); + setLastRefresh(Date.now()); + }); + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }, []); + + useEffect(() => { + if (globalState?.refreshInterval !== undefined) { + timefilter.setRefreshInterval(globalState.refreshInterval); + } + }, [JSON.stringify(globalState?.refreshInterval), timefilter]); + + const { + documentCountStats, + totalCount, + overallStats, + overallStatsProgress, + columns, + cancelOverallStatsRequest, + } = useESQLOverallStatsData(fieldStatsRequest); + + const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); + const [metricsLoaded] = useState(defaults.metricsLoaded); + const [metricsStats, setMetricsStats] = useState(); + + const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); + const [nonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + + const [fieldStatFieldsToFetch, setFieldStatFieldsToFetch] = useState(); + + const visibleFieldTypes = + dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes; + + const visibleFieldNames = + dataVisualizerListState.visibleFieldNames ?? restorableDefaults.visibleFieldNames; + + useEffect( + function updateFieldStatFieldsToFetch() { + const { sortField, sortDirection } = dataVisualizerListState; + + // Otherwise, sort the list of fields by the initial sort field and sort direction + // Then divide into chunks by the initial page size + + const itemsSorter = Comparators.property( + sortField as string, + Comparators.default(sortDirection as 'asc' | 'desc' | undefined) + ); + + const preslicedSortedConfigs = [...nonMetricConfigs, ...metricConfigs] + .map((c) => ({ + ...c, + name: c.fieldName, + docCount: c.stats?.count, + cardinality: c.stats?.cardinality, + })) + .sort(itemsSorter); + + const filteredItems = filterFields( + preslicedSortedConfigs, + dataVisualizerListState.visibleFieldNames, + dataVisualizerListState.visibleFieldTypes + ); + + const { pageIndex, pageSize } = dataVisualizerListState; + + const pageOfConfigs = filteredItems.filteredFields + ?.slice(pageIndex * pageSize, (pageIndex + 1) * pageSize) + .filter((d) => d.existsInDocs === true); + + setFieldStatFieldsToFetch(pageOfConfigs); + }, + [ + dataVisualizerListState.pageIndex, + dataVisualizerListState.pageSize, + dataVisualizerListState.sortField, + dataVisualizerListState.sortDirection, + nonMetricConfigs, + metricConfigs, + ] + ); + + const { fieldStats, fieldStatsProgress, cancelFieldStatsRequest } = useESQLFieldStatsData({ + searchQuery: fieldStatsRequest?.searchQuery, + columns: fieldStatFieldsToFetch, + filter: fieldStatsRequest?.filter, + limitSize: fieldStatsRequest?.limitSize, + }); + + const createMetricCards = useCallback(() => { + if (!columns || !overallStats) return; + const configs: FieldVisConfig[] = []; + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; + + const allMetricFields = columns.filter((f) => { + return f.secondaryType === KBN_FIELD_TYPES.NUMBER; + }); + + const metricExistsFields = allMetricFields.filter((f) => { + return aggregatableExistsFields.find((existsF) => { + return existsF.fieldName === f.name; + }); + }); + + let _aggregatableFields: AggregatableField[] = overallStats.aggregatableExistsFields; + if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { + _aggregatableFields = _aggregatableFields.concat(overallStats.aggregatableNotExistsFields); + } + + const metricFieldsToShow = + metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; + + metricFieldsToShow.forEach((field) => { + const fieldData = _aggregatableFields.find((f) => { + return f.fieldName === field.name; + }); + if (!fieldData) return; + + const metricConfig: FieldVisConfig = { + ...field, + ...fieldData, + loading: fieldData?.existsInDocs ?? true, + fieldFormat: + currentDataView?.getFormatterForFieldNoDefault(field.name) ?? + fieldFormats.deserialize({ id: field.secondaryType }), + aggregatable: true, + deletable: false, + type: getFieldType(field) as SupportedFieldType, + }; + + configs.push(metricConfig); + }); + + setMetricsStats({ + totalMetricFieldsCount: allMetricFields.length, + visibleMetricsCount: metricFieldsToShow.length, + }); + setMetricConfigs(configs); + }, [metricsLoaded, overallStats, showEmptyFields, columns, currentDataView?.id]); + + const createNonMetricCards = useCallback(() => { + if (!columns || !overallStats) return; + + const allNonMetricFields = columns.filter((f) => { + return f.secondaryType !== KBN_FIELD_TYPES.NUMBER; + }); + // Obtain the list of all non-metric fields which appear in documents + // (aggregatable or not aggregatable). + const populatedNonMetricFields: Column[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: Array = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: NonAggregatableField[] = + overallStats.nonAggregatableExistsFields || []; + + allNonMetricFields.forEach((f) => { + const checkAggregatableField = aggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.name + ); + + if (checkAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkAggregatableField); + } else { + const checkNonAggregatableField = nonAggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.name + ); + + if (checkNonAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkNonAggregatableField); + } + } + }); + + if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { + // Combine the field data obtained from Elasticsearch into a single array. + nonMetricFieldData = nonMetricFieldData.concat( + overallStats.aggregatableNotExistsFields, + overallStats.nonAggregatableNotExistsFields + ); + } + + const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; + + const configs: FieldVisConfig[] = []; + + nonMetricFieldsToShow.forEach((field) => { + const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.name); + const nonMetricConfig: Partial = { + ...(fieldData ? fieldData : {}), + secondaryType: getFieldType(field) as SupportedFieldType, + loading: fieldData?.existsInDocs ?? true, + deletable: false, + fieldFormat: + currentDataView?.getFormatterForFieldNoDefault(field.name) ?? + fieldFormats.deserialize({ id: field.secondaryType }), + }; + + // Map the field type from the Kibana index pattern to the field type + // used in the data visualizer. + const dataVisualizerType = getFieldType(field) as SupportedFieldType; + if (dataVisualizerType !== undefined) { + nonMetricConfig.type = dataVisualizerType; + } else { + // Add a flag to indicate that this is one of the 'other' Kibana + // field types that do not yet have a specific card type. + nonMetricConfig.type = field.type as SupportedFieldType; + nonMetricConfig.isUnsupportedType = true; + } + + if (field.name !== nonMetricConfig.fieldName) { + nonMetricConfig.displayName = field.name; + } + + configs.push(nonMetricConfig as FieldVisConfig); + }); + + setNonMetricConfigs(configs); + }, [columns, nonMetricsLoaded, overallStats, showEmptyFields, currentDataView?.id]); + + const fieldsCountStats: TotalFieldsStats | undefined = useMemo(() => { + if (!overallStats) return; + + let _visibleFieldsCount = 0; + let _totalFieldsCount = 0; + Object.keys(overallStats).forEach((key) => { + const fieldsGroup = overallStats[key as keyof typeof overallStats]; + if (Array.isArray(fieldsGroup) && fieldsGroup.length > 0) { + _totalFieldsCount += fieldsGroup.length; + } + }); + + if (showEmptyFields === true) { + _visibleFieldsCount = _totalFieldsCount; + } else { + _visibleFieldsCount = + overallStats.aggregatableExistsFields.length + + overallStats.nonAggregatableExistsFields.length; + } + return { visibleFieldsCount: _visibleFieldsCount, totalFieldsCount: _totalFieldsCount }; + }, [overallStats, showEmptyFields]); + + useEffect(() => { + createMetricCards(); + createNonMetricCards(); + }, [overallStats, showEmptyFields]); + + const configs = useMemo(() => { + let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; + + combinedConfigs = filterFields( + combinedConfigs, + visibleFieldNames, + visibleFieldTypes + ).filteredFields; + + if (fieldStatsProgress.loaded === 100 && fieldStats) { + combinedConfigs = combinedConfigs.map((c) => { + const loadedFullStats = fieldStats.get(c.fieldName) ?? {}; + return loadedFullStats + ? { + ...c, + loading: false, + stats: { ...c.stats, ...loadedFullStats }, + } + : c; + }); + } + return combinedConfigs; + }, [ + nonMetricConfigs, + metricConfigs, + visibleFieldTypes, + visibleFieldNames, + fieldStatsProgress.loaded, + dataVisualizerListState.pageIndex, + dataVisualizerListState.pageSize, + ]); + + // Some actions open up fly-out or popup + // This variable is used to keep track of them and clean up when unmounting + const actionFlyoutRef = useRef<() => void | undefined>(); + useEffect(() => { + const ref = actionFlyoutRef; + return () => { + // Clean up any of the flyout/editor opened from the actions + if (ref.current) { + ref.current(); + } + }; + }, []); + + const getItemIdToExpandedRowMap = useCallback( + function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { + return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { + const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); + if (item !== undefined) { + m[fieldName] = ( + + ); + } + return m; + }, {} as ItemIdToExpandedRowMap); + }, + [currentDataView, totalCount] + ); + + const hasValidTimeField = useMemo( + () => + currentDataView && + currentDataView.timeFieldName !== undefined && + currentDataView.timeFieldName !== '', + [currentDataView] + ); + + const isWithinLargeBreakpoint = useIsWithinMaxBreakpoint('l'); + const dvPageHeader = css({ + [useEuiBreakpoint(['xs', 's', 'm', 'l'])]: { + flexDirection: 'column', + alignItems: 'flex-start', + }, + }); + + const combinedProgress = useMemo( + () => overallStatsProgress.loaded * 0.3 + fieldStatsProgress.loaded * 0.7, + [overallStatsProgress.loaded, fieldStatsProgress.loaded] + ); + + // Query that has been typed, but has not submitted with cmd + enter + const [localQuery, setLocalQuery] = useState({ esql: '' }); + + const onQueryUpdate = (q?: AggregateQuery) => { + // When user submits a new query + // resets all current requests and other data + if (cancelOverallStatsRequest) { + cancelOverallStatsRequest(); + } + if (cancelFieldStatsRequest) { + cancelFieldStatsRequest(); + } + // Reset field stats to fetch state + setFieldStatFieldsToFetch(undefined); + setMetricConfigs(defaults.metricConfigs); + setNonMetricConfigs(defaults.nonMetricConfigs); + if (q) { + setQuery(q); + } + }; + + useEffect( + function resetFieldStatsFieldToFetch() { + // If query returns 0 document, no need to do more work here + if (totalCount === undefined || totalCount === 0) { + setFieldStatFieldsToFetch(undefined); + return; + } + }, + [totalCount] + ); + + return ( + + + + + + {isWithinLargeBreakpoint ? : null} + + {hasValidTimeField && currentDataView ? ( + + {}} + dataView={currentDataView} + query={undefined} + disabled={false} + timefilter={timefilter} + /> + + ) : null} + + + + + + + false} + isCodeEditorExpanded={true} + detectTimestamp={true} + hideMinimizeButton={true} + hideRunQueryText={false} + /> + + + + + {totalCount !== undefined && ( + <> + + + + + )} + + + + + + + + + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + loading={overallStatsProgress.isRunning} + overallStatsRunning={overallStatsProgress.isRunning} + showPreviewByDefault={dataVisualizerListState.showDistributions ?? true} + onChange={setDataVisualizerListState} + totalCount={totalCount} + /> + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index d752cb4b166f59..cc17387886071c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -64,9 +64,9 @@ import { SearchPanel } from '../search_panel'; import { ActionsPanel } from '../actions_panel'; import { createMergedEsQuery } from '../../utils/saved_search_utils'; import { DataVisualizerDataViewManagement } from '../data_view_management'; -import { GetAdditionalLinks } from '../../../common/components/results_links'; +import type { GetAdditionalLinks } from '../../../common/components/results_links'; import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; -import { DataVisualizerGridInput } from '../../embeddables/grid_embeddable/grid_embeddable'; +import type { DataVisualizerGridInput } from '../../embeddables/grid_embeddable/grid_embeddable'; import { MIN_SAMPLER_PROBABILITY, RANDOM_SAMPLER_OPTION, @@ -115,7 +115,6 @@ export const getDefaultDataVisualizerListState = ( sortDirection: 'asc', visibleFieldTypes: [], visibleFieldNames: [], - samplerShardSize: 5000, searchString: '', searchQuery: defaultSearchQuery, searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx new file mode 100644 index 00000000000000..bcdf3241f5ee3e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { type ChangeEvent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiSelect, EuiText, useGeneratedHtmlId } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; + +const options = [ + { + value: '5000', + text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', { + defaultMessage: '{limit} rows', + values: { limit: '5,000' }, + }), + }, + { + value: '10000', + text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', { + defaultMessage: '{limit} rows', + values: { limit: '10,000' }, + }), + }, + { + value: '100000', + text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', { + defaultMessage: '{limit} rows', + values: { limit: '100,000' }, + }), + }, + { + value: '1000000', + text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', { + defaultMessage: '{limit} rows', + values: { limit: '1,000,000' }, + }), + }, + { + value: 'none', + text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.analyzeAll', { + defaultMessage: 'Analyze all', + }), + }, +]; + +export type ESQLDefaultLimitSizeOption = '5000' | '10000' | '100000' | '1000000' | 'none'; + +export const ESQLDefaultLimitSizeSelect = ({ + limitSize, + onChangeLimitSize, +}: { + limitSize: string; + onChangeLimitSize: (newLimit: ESQLDefaultLimitSizeOption) => void; +}) => { + const basicSelectId = useGeneratedHtmlId({ prefix: 'dvESQLLimit' }); + + const onChange = (e: ChangeEvent) => { + onChangeLimitSize(e.target.value as ESQLDefaultLimitSizeOption); + }; + + return ( + + + + } + /> + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_field_stats_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_field_stats_data.ts new file mode 100644 index 00000000000000..5c034bf82ebf7e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_field_stats_data.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import type { AggregateQuery } from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import { useEffect, useReducer, useState } from 'react'; +import { chunk } from 'lodash'; +import { useCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { DataStatsFetchProgress, FieldStats } from '../../../../../common/types/field_stats'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { getInitialProgress, getReducer } from '../../progress_utils'; +import { isESQLQuery, getSafeESQLLimitSize } from '../../search_strategy/requests/esql_utils'; +import type { Column } from './use_esql_overall_stats_data'; +import { getESQLNumericFieldStats } from '../../search_strategy/esql_requests/get_numeric_field_stats'; +import { getESQLKeywordFieldStats } from '../../search_strategy/esql_requests/get_keyword_fields'; +import { getESQLDateFieldStats } from '../../search_strategy/esql_requests/get_date_field_stats'; +import { getESQLBooleanFieldStats } from '../../search_strategy/esql_requests/get_boolean_field_stats'; +import { getESQLTextFieldStats } from '../../search_strategy/esql_requests/get_text_field_stats'; + +export const useESQLFieldStatsData = ({ + searchQuery, + columns: allColumns, + filter, + limitSize, +}: { + searchQuery?: AggregateQuery; + columns?: T[]; + filter?: QueryDslQueryContainer; + limitSize?: string; +}) => { + const [fieldStats, setFieldStats] = useState>(); + + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const { + services: { + data, + notifications: { toasts }, + }, + } = useDataVisualizerKibana(); + + const { runRequest, cancelRequest } = useCancellableSearch(data); + + useEffect( + function updateFieldStats() { + let unmounted = false; + + const fetchFieldStats = async () => { + cancelRequest(); + + if (!isESQLQuery(searchQuery) || !allColumns) return; + + setFetchState({ + ...getInitialProgress(), + isRunning: true, + error: undefined, + }); + try { + // By default, limit the source data to 100,000 rows + const esqlBaseQuery = searchQuery.esql + getSafeESQLLimitSize(limitSize); + + const totalFieldsCnt = allColumns.length; + const processedFieldStats = new Map(); + + function addToProcessedFieldStats(stats: Array) { + if (!unmounted) { + stats.forEach((field) => { + if (field) { + processedFieldStats.set(field.fieldName!, field); + } + }); + setFetchState({ + loaded: (processedFieldStats.size / totalFieldsCnt) * 100, + }); + } + } + setFieldStats(processedFieldStats); + + const aggregatableFieldsChunks = chunk(allColumns, 25); + + for (const columns of aggregatableFieldsChunks) { + // GETTING STATS FOR NUMERIC FIELDS + await getESQLNumericFieldStats({ + columns: columns.filter((f) => f.secondaryType === 'number'), + filter, + runRequest, + esqlBaseQuery, + }).then(addToProcessedFieldStats); + + // GETTING STATS FOR KEYWORD FIELDS + await getESQLKeywordFieldStats({ + columns: columns.filter( + (f) => f.secondaryType === 'keyword' || f.secondaryType === 'ip' + ), + filter, + runRequest, + esqlBaseQuery, + }).then(addToProcessedFieldStats); + + // GETTING STATS FOR BOOLEAN FIELDS + await getESQLBooleanFieldStats({ + columns: columns.filter((f) => f.secondaryType === 'boolean'), + filter, + runRequest, + esqlBaseQuery, + }).then(addToProcessedFieldStats); + + // GETTING STATS FOR TEXT FIELDS + await getESQLTextFieldStats({ + columns: columns.filter((f) => f.secondaryType === 'text'), + filter, + runRequest, + esqlBaseQuery, + }).then(addToProcessedFieldStats); + + // GETTING STATS FOR DATE FIELDS + await getESQLDateFieldStats({ + columns: columns.filter((f) => f.secondaryType === 'date'), + filter, + runRequest, + esqlBaseQuery, + }).then(addToProcessedFieldStats); + } + setFetchState({ + loaded: 100, + isRunning: false, + }); + } catch (e) { + if (e.name !== 'AbortError') { + const title = i18n.translate( + 'xpack.dataVisualizer.index.errorFetchingESQLFieldStatisticsMessage', + { + defaultMessage: 'Error fetching field statistics for ES|QL query', + } + ); + toasts.addError(e, { + title, + }); + + // Log error to console for better debugging + // eslint-disable-next-line no-console + console.error(`${title}: fetchFieldStats`, e); + setFetchState({ + loaded: 100, + isRunning: false, + error: e, + }); + } + } + }; + fetchFieldStats(); + + return () => { + unmounted = true; + }; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [allColumns, JSON.stringify({ filter }), limitSize] + ); + + return { fieldStats, fieldStatsProgress: fetchState, cancelFieldStatsRequest: cancelRequest }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts new file mode 100644 index 00000000000000..c23a75e2e3bac6 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts @@ -0,0 +1,395 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQL_SEARCH_STRATEGY, KBN_FIELD_TYPES } from '@kbn/data-plugin/common'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import type { AggregateQuery } from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import { useCallback, useEffect, useMemo, useReducer } from 'react'; +import { type UseCancellableSearch, useCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { estypes } from '@elastic/elasticsearch'; +import type { ISearchOptions } from '@kbn/data-plugin/common'; +import { OMIT_FIELDS } from '../../../../../common/constants'; +import type { TimeBucketsInterval } from '../../../../../common/services/time_buckets'; +import type { + DataStatsFetchProgress, + DocumentCountStats, +} from '../../../../../common/types/field_stats'; +import { getSupportedFieldType } from '../../../common/components/fields_stats_grid/get_field_names'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { getInitialProgress, getReducer } from '../../progress_utils'; +import { + getSafeESQLLimitSize, + getSafeESQLName, + isESQLQuery, +} from '../../search_strategy/requests/esql_utils'; +import type { NonAggregatableField } from '../../types/overall_stats'; +import { getESQLSupportedAggs } from '../../utils/get_supported_aggs'; +import type { ESQLDefaultLimitSizeOption } from '../../components/search_panel/esql/limit_size'; +import { getESQLOverallStats } from '../../search_strategy/esql_requests/get_count_and_cardinality'; +import type { AggregatableField } from '../../types/esql_data_visualizer'; +import { + handleError, + type HandleErrorCallback, +} from '../../search_strategy/esql_requests/handle_error'; + +export interface Column { + type: string; + name: string; + secondaryType: string; +} + +interface Data { + timeFieldName?: string; + columns?: Column[]; + totalCount?: number; + nonAggregatableFields?: Array<{ name: string; type: string }>; + aggregatableFields?: Array<{ name: string; type: string; supportedAggs: Set }>; + documentCountStats?: DocumentCountStats; + overallStats?: { + aggregatableExistsFields: AggregatableField[]; + aggregatableNotExistsFields: AggregatableField[]; + nonAggregatableExistsFields: NonAggregatableField[]; + nonAggregatableNotExistsFields: NonAggregatableField[]; + }; +} + +const getESQLDocumentCountStats = async ( + runRequest: UseCancellableSearch['runRequest'], + query: AggregateQuery, + filter?: estypes.QueryDslQueryContainer, + timeFieldName?: string, + intervalMs?: number, + searchOptions?: ISearchOptions, + onError?: HandleErrorCallback +): Promise<{ documentCountStats?: DocumentCountStats; totalCount: number }> => { + if (!isESQLQuery(query)) { + throw Error( + i18n.translate('xpack.dataVisualizer.esql.noQueryProvided', { + defaultMessage: 'No ES|QL query provided', + }) + ); + } + const esqlBaseQuery = query.esql; + let earliestMs = Infinity; + let latestMs = -Infinity; + + if (timeFieldName) { + const aggQuery = ` | EVAL _timestamp_= TO_DOUBLE(DATE_TRUNC(${intervalMs} millisecond, ${getSafeESQLName( + timeFieldName + )})) + | stats rows = count(*) by _timestamp_ + | LIMIT 10000`; + + const request = { + params: { + query: esqlBaseQuery + aggQuery, + ...(filter ? { filter } : {}), + }, + }; + try { + const esqlResults = await runRequest(request, { ...(searchOptions ?? {}), strategy: 'esql' }); + let totalCount = 0; + const _buckets: Record = {}; + // @ts-expect-error ES types needs to be updated with columns and values as part of esql response + esqlResults?.rawResponse.values.forEach((val) => { + const [count, bucket] = val; + _buckets[bucket] = count; + totalCount += count; + if (bucket < earliestMs) { + earliestMs = bucket; + } + if (bucket >= latestMs) { + latestMs = bucket; + } + }); + const result: DocumentCountStats = { + interval: intervalMs, + probability: 1, + randomlySampled: false, + timeRangeEarliest: earliestMs, + timeRangeLatest: latestMs, + buckets: _buckets, + totalCount, + }; + return { documentCountStats: result, totalCount }; + } catch (error) { + handleError({ + request, + error, + onError, + title: i18n.translate('xpack.dataVisualizer.esql.docCountError', { + defaultMessage: `Error getting total count & doc count chart for ES|QL time-series data for request:`, + }), + }); + return Promise.reject(error); + } + } else { + // If not time field, get the total count + const request = { + params: { + query: esqlBaseQuery + ' | STATS _count_ = COUNT(*) | LIMIT 1', + ...(filter ? { filter } : {}), + }, + }; + try { + const esqlResults = await runRequest(request, { ...(searchOptions ?? {}), strategy: 'esql' }); + return { + documentCountStats: undefined, + totalCount: esqlResults?.rawResponse.values[0][0], + }; + } catch (error) { + handleError({ + request, + error, + onError, + title: i18n.translate('xpack.dataVisualizer.esql.docCountNoneTimeseriesError', { + defaultMessage: `Error getting total count for ES|QL data:`, + }), + }); + return Promise.reject(error); + } + } +}; + +export const getInitialData = (): Data => ({ + timeFieldName: undefined, + columns: undefined, + totalCount: undefined, +}); + +const NON_AGGREGATABLE_FIELD_TYPES = new Set([ + KBN_FIELD_TYPES.GEO_SHAPE, + KBN_FIELD_TYPES.GEO_POINT, + KBN_FIELD_TYPES.HISTOGRAM, +]); + +const fieldStatsErrorTitle = i18n.translate( + 'xpack.dataVisualizer.index.errorFetchingESQLFieldStatisticsMessage', + { + defaultMessage: 'Error fetching field statistics for ES|QL query', + } +); + +export const useESQLOverallStatsData = ( + fieldStatsRequest: + | { + earliest: number | undefined; + latest: number | undefined; + aggInterval: TimeBucketsInterval; + intervalMs: number; + searchQuery: AggregateQuery; + indexPattern: string | undefined; + timeFieldName: string | undefined; + lastRefresh: number; + filter?: QueryDslQueryContainer; + limitSize?: ESQLDefaultLimitSizeOption; + } + | undefined +) => { + const { + services: { + data, + notifications: { toasts }, + }, + } = useDataVisualizerKibana(); + + const { runRequest, cancelRequest } = useCancellableSearch(data); + + const [tableData, setTableData] = useReducer(getReducer(), getInitialData()); + const [overallStatsProgress, setOverallStatsProgress] = useReducer( + getReducer(), + getInitialProgress() + ); + const onError = useCallback( + (error, title?: string) => + toasts.addError(error, { + title: title ?? fieldStatsErrorTitle, + }), + [toasts] + ); + + const startFetch = useCallback( + async function fetchOverallStats() { + try { + cancelRequest(); + + if (!fieldStatsRequest) { + return; + } + setOverallStatsProgress({ + ...getInitialProgress(), + isRunning: true, + error: undefined, + }); + setTableData({ totalCount: undefined, documentCountStats: undefined }); + + const { searchQuery, intervalMs, filter, limitSize } = fieldStatsRequest; + + if (!isESQLQuery(searchQuery)) { + return; + } + + const intervalInMs = intervalMs === 0 ? 60 * 60 * 60 * 10 : intervalMs; + + // For doc count chart, we want the full base query without any limit + const esqlBaseQuery = searchQuery.esql; + + const columnsResp = await runRequest( + { + params: { + query: esqlBaseQuery + '| LIMIT 0', + ...(filter ? { filter } : {}), + }, + }, + { strategy: ESQL_SEARCH_STRATEGY } + ); + const columns = columnsResp?.rawResponse + ? // @ts-expect-error ES types need to be updated with columns for ESQL queries + (columnsResp.rawResponse.columns.map((c) => ({ + ...c, + secondaryType: getSupportedFieldType(c.type), + })) as Column[]) + : []; + + const timeFields = columns.filter((d) => d.type === 'date'); + + const dataViewTimeField = timeFields.find( + (f) => f.name === fieldStatsRequest?.timeFieldName + ) + ? fieldStatsRequest?.timeFieldName + : undefined; + + // If a date field named '@timestamp' exists, set that as default time field + // Else, use the default time view defined by data view + // Else, use first available date field as default + const timeFieldName = + timeFields.length > 0 + ? timeFields.find((f) => f.name === '@timestamp') + ? '@timestamp' + : dataViewTimeField ?? timeFields[0].name + : undefined; + + setTableData({ columns, timeFieldName }); + + const { totalCount, documentCountStats } = await getESQLDocumentCountStats( + runRequest, + searchQuery, + filter, + timeFieldName, + intervalInMs, + undefined, + onError + ); + + setTableData({ totalCount, documentCountStats }); + setOverallStatsProgress({ + loaded: 50, + }); + const aggregatableFields: Array<{ + fieldName: string; + name: string; + type: string; + supportedAggs: Set; + secondaryType: string; + aggregatable: boolean; + }> = []; + const nonAggregatableFields: Array<{ + fieldName: string; + name: string; + type: string; + secondaryType: string; + }> = []; + const fields = columns + // Some field types are not supported by ESQL yet + // Also, temporarily removing null columns because it causes problems with some aggs + // See https://github.com/elastic/elasticsearch/issues/104430 + .filter((c) => c.type !== 'unsupported' && c.type !== 'null') + .map((field) => { + return { ...field, aggregatable: !NON_AGGREGATABLE_FIELD_TYPES.has(field.type) }; + }); + + fields?.forEach((field) => { + const fieldName = field.name; + if (!OMIT_FIELDS.includes(fieldName)) { + if (!field.aggregatable) { + nonAggregatableFields.push({ + ...field, + fieldName: field.name, + secondaryType: getSupportedFieldType(field.type), + }); + } else { + aggregatableFields.push({ + ...field, + fieldName: field.name, + secondaryType: getSupportedFieldType(field.type), + supportedAggs: getESQLSupportedAggs(field, true), + aggregatable: true, + }); + } + } + }); + + setTableData({ aggregatableFields, nonAggregatableFields }); + + // COUNT + CARDINALITY + // For % count & cardinality, we want the full base query WITH specified limit + // to safeguard against huge datasets + const esqlBaseQueryWithLimit = searchQuery.esql + getSafeESQLLimitSize(limitSize); + + if (totalCount === 0) { + setOverallStatsProgress({ + loaded: 100, + isRunning: false, + error: undefined, + }); + return; + } + if (totalCount > 0 && fields.length > 0) { + const stats = await getESQLOverallStats({ + runRequest, + fields, + esqlBaseQueryWithLimit, + filter, + limitSize, + totalCount, + onError, + }); + + setTableData({ overallStats: stats }); + setOverallStatsProgress({ + loaded: 100, + isRunning: false, + error: undefined, + }); + } + } catch (error) { + // If error already handled in sub functions, no need to propogate + if (error.name !== 'AbortError' && error.handled !== true) { + toasts.addError(error, { + title: fieldStatsErrorTitle, + }); + // Log error to console for better debugging + // eslint-disable-next-line no-console + console.error(`${fieldStatsErrorTitle}: fetchOverallStats`, error); + } + } + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [runRequest, toasts, JSON.stringify({ fieldStatsRequest }), onError] + ); + + // auto-update + useEffect(() => { + startFetch(); + }, [startFetch]); + + return useMemo( + () => ({ ...tableData, overallStatsProgress, cancelOverallStatsRequest: cancelRequest }), + [tableData, overallStatsProgress, cancelRequest] + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index e4ba7c1ee9050c..b012d049ae04f1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -10,7 +10,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { merge } from 'rxjs'; import type { EuiTableActionsColumnType } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { type DataViewField } from '@kbn/data-plugin/common'; +import { UI_SETTINGS, type DataViewField } from '@kbn/data-plugin/common'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import seedrandom from 'seedrandom'; import type { SamplingOption } from '@kbn/discover-plugin/public/application/main/components/field_stats_table/field_stats_table'; @@ -44,6 +44,7 @@ import { useOverallStats } from './use_overall_stats'; import type { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; import type { AggregatableField, NonAggregatableField } from '../types/overall_stats'; import { getSupportedAggs } from '../utils/get_supported_aggs'; +import { DEFAULT_BAR_TARGET } from '../../common/constants'; const defaults = getDefaultPageState(); @@ -83,7 +84,7 @@ export const useDataVisualizerGridData = ( useExecutionContext(executionContext, embeddableExecutionContext); - const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + const { visibleFieldTypes, showEmptyFields } = dataVisualizerListState; const [lastRefresh, setLastRefresh] = useState(0); const searchSessionId = input.sessionId; @@ -205,12 +206,12 @@ export const useDataVisualizerGridData = ( } const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; + const barTarget = uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET) ?? DEFAULT_BAR_TARGET; buckets.setInterval('auto'); if (bounds) { buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); + buckets.setBarTarget(barTarget); } const aggInterval = buckets.getInterval(); @@ -243,7 +244,6 @@ export const useDataVisualizerGridData = ( aggInterval, intervalMs: aggInterval?.asMilliseconds(), searchQuery, - samplerShardSize, sessionId: searchSessionId, index: currentDataView.title, timeFieldName: currentDataView.timeFieldName, @@ -265,7 +265,6 @@ export const useDataVisualizerGridData = ( JSON.stringify(searchQuery), // eslint-disable-next-line react-hooks/exhaustive-deps JSON.stringify(samplingOption), - samplerShardSize, searchSessionId, lastRefresh, fieldsToFetch, @@ -275,6 +274,7 @@ export const useDataVisualizerGridData = ( ); const { overallStats, progress: overallStatsProgress } = useOverallStats( + false, fieldStatsRequest, lastRefresh, dataVisualizerListState.probability diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index f47bd9aebb33d0..16efc0ef6f1ea1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -73,6 +73,7 @@ export function useFieldStatsSearchStrategy( } = useDataVisualizerKibana(); const [fieldStats, setFieldStats] = useState>(); + const [fetchState, setFetchState] = useReducer( getReducer(), getInitialProgress() @@ -154,7 +155,6 @@ export function useFieldStatsSearchStrategy( const params: FieldStatsCommonRequestParams = { index: searchStrategyParams.index, - samplerShardSize: searchStrategyParams.samplerShardSize, timeFieldName: searchStrategyParams.timeFieldName, earliestMs: searchStrategyParams.earliest, latestMs: searchStrategyParams.latest, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 53fe7d8b1cafd2..ff6030c45f96ec 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -66,6 +66,7 @@ export function rateLimitingForkJoin( } export function useOverallStats( + esql = false, searchStrategyParams: TParams | undefined, lastRefresh: number, probability?: number | null diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 6b4a401b106295..0ab275e03fc1f4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -26,6 +26,7 @@ import { type Accessor, type Dictionary, type SetUrlState, + UrlStateProvider, } from '@kbn/ml-url-state'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import { getCoreStart, getPluginsStart } from '../../kibana_services'; @@ -33,6 +34,8 @@ import { type IndexDataVisualizerViewProps, IndexDataVisualizerView, } from './components/index_data_visualizer_view'; +import { IndexDataVisualizerESQL } from './components/index_data_visualizer_view/index_data_visualizer_esql'; + import { useDataVisualizerKibana } from '../kibana_context'; import type { GetAdditionalLinks } from '../common/components/results_links'; import { DATA_VISUALIZER_APP_LOCATOR, type IndexDataVisualizerLocatorParams } from './locator'; @@ -80,7 +83,15 @@ export const getLocatorParams = (params: { return locatorParams; }; -export const DataVisualizerStateContextProvider: FC = ({ +const DataVisualizerESQLStateContextProvider = () => { + return ( + + + + ); +}; + +const DataVisualizerStateContextProvider: FC = ({ IndexDataVisualizerComponent, getAdditionalLinks, }) => { @@ -256,9 +267,7 @@ export const DataVisualizerStateContextProvider: FC - ) : ( -
- )} + ) : null} ); }; @@ -266,11 +275,13 @@ export const DataVisualizerStateContextProvider: FC = ({ getAdditionalLinks, showFrozenDataTierChoice = true, + esql, }) => { const coreStart = getCoreStart(); const { @@ -320,10 +331,14 @@ export const IndexDataVisualizer: FC = ({ - + {!esql ? ( + + ) : ( + + )} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts new file mode 100644 index 00000000000000..b4adcd4ee4f05b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import pLimit from 'p-limit'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import { getSafeESQLName } from '../requests/esql_utils'; +import { isFulfilled, isRejected } from '../../../common/util/promise_all_settled_utils'; +import { MAX_CONCURRENT_REQUESTS } from '../../constants/index_data_visualizer_viewer'; +import type { BucketCount } from '../../types/esql_data_visualizer'; +import type { BooleanFieldStats, FieldStatsError } from '../../../../../common/types/field_stats'; + +interface Params { + runRequest: UseCancellableSearch['runRequest']; + columns: Column[]; + esqlBaseQuery: string; + filter?: QueryDslQueryContainer; +} + +export const getESQLBooleanFieldStats = async ({ + runRequest, + columns, + esqlBaseQuery, + filter, +}: Params): Promise> => { + const limiter = pLimit(MAX_CONCURRENT_REQUESTS); + + const booleanFields = columns + .filter((f) => f.secondaryType === 'boolean') + .map((field) => { + const query = `| STATS ${getSafeESQLName(`${field.name}_terms`)} = count(${getSafeESQLName( + field.name + )}) BY ${getSafeESQLName(field.name)} + | LIMIT 3`; + + return { + field, + request: { + params: { + query: esqlBaseQuery + query, + ...(filter ? { filter } : {}), + }, + }, + }; + }); + + if (booleanFields.length > 0) { + const booleanTopTermsResp = await Promise.allSettled( + booleanFields.map(({ request }) => + limiter(() => runRequest(request, { strategy: ESQL_SEARCH_STRATEGY })) + ) + ); + if (booleanTopTermsResp) { + return booleanFields.map(({ field, request }, idx) => { + const resp = booleanTopTermsResp[idx]; + + if (!resp) return; + + if (isFulfilled(resp) && resp.value) { + const results = resp.value.rawResponse.values as Array<[BucketCount, boolean]>; + const topValuesSampleSize = results.reduce((acc, row) => acc + row[0], 0); + + let falseCount = 0; + let trueCount = 0; + const terms = results.map((row) => { + if (row[1] === false) { + falseCount = row[0]; + } + if (row[1] === true) { + trueCount = row[0]; + } + return { + key_as_string: row[1]?.toString(), + doc_count: row[0], + percent: row[0] / topValuesSampleSize, + }; + }); + + return { + fieldName: field.name, + topValues: terms, + topValuesSampleSize, + topValuesSamplerShardSize: topValuesSampleSize, + isTopValuesSampled: false, + trueCount, + falseCount, + count: trueCount + falseCount, + } as BooleanFieldStats; + } + + if (isRejected(resp)) { + // Log for debugging purposes + // eslint-disable-next-line no-console + console.error(resp, request); + + return { + fieldName: field.name, + error: resp.reason, + } as FieldStatsError; + } + }); + } + } + return []; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts new file mode 100644 index 00000000000000..72e67db1fafaed --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts @@ -0,0 +1,200 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import pLimit from 'p-limit'; +import { chunk } from 'lodash'; +import { isDefined } from '@kbn/ml-is-defined'; +import type { ESQLSearchReponse } from '@kbn/es-types'; +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { i18n } from '@kbn/i18n'; +import { getSafeESQLName } from '../requests/esql_utils'; +import { MAX_CONCURRENT_REQUESTS } from '../../constants/index_data_visualizer_viewer'; +import type { NonAggregatableField } from '../../types/overall_stats'; +import { isFulfilled } from '../../../common/util/promise_all_settled_utils'; +import type { ESQLDefaultLimitSizeOption } from '../../components/search_panel/esql/limit_size'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import { AggregatableField } from '../../types/esql_data_visualizer'; +import { handleError, HandleErrorCallback } from './handle_error'; + +interface Field extends Column { + aggregatable?: boolean; +} +const getESQLOverallStatsInChunk = async ({ + runRequest, + fields, + esqlBaseQueryWithLimit, + filter, + limitSize, + totalCount, + onError, +}: { + runRequest: UseCancellableSearch['runRequest']; + fields: Field[]; + esqlBaseQueryWithLimit: string; + filter?: estypes.QueryDslQueryContainer; + limitSize?: ESQLDefaultLimitSizeOption; + totalCount: number; + onError?: HandleErrorCallback; +}) => { + if (fields.length > 0) { + const aggregatableFieldsToQuery = fields.filter((f) => f.aggregatable); + + let countQuery = aggregatableFieldsToQuery.length > 0 ? '| STATS ' : ''; + countQuery += aggregatableFieldsToQuery + .map((field) => { + // count idx = 0, cardinality idx = 1 + return `${getSafeESQLName(`${field.name}_count`)} = COUNT(${getSafeESQLName(field.name)}), + ${getSafeESQLName(`${field.name}_cardinality`)} = COUNT_DISTINCT(${getSafeESQLName( + field.name + )})`; + }) + .join(','); + + const request = { + params: { + query: esqlBaseQueryWithLimit + countQuery, + ...(filter ? { filter } : {}), + }, + }; + try { + const esqlResults = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + const stats = { + aggregatableExistsFields: [] as AggregatableField[], + aggregatableNotExistsFields: [] as AggregatableField[], + nonAggregatableExistsFields: [] as NonAggregatableField[], + nonAggregatableNotExistsFields: [] as NonAggregatableField[], + }; + + if (!esqlResults) { + return; + } + const esqlResultsResp = esqlResults.rawResponse as unknown as ESQLSearchReponse; + + const sampleCount = + limitSize === 'none' || !isDefined(limitSize) ? totalCount : parseInt(limitSize, 10); + aggregatableFieldsToQuery.forEach((field, idx) => { + const count = esqlResultsResp.values[0][idx * 2] as number; + const cardinality = esqlResultsResp.values[0][idx * 2 + 1] as number; + + if (field.aggregatable === true) { + if (count > 0) { + stats.aggregatableExistsFields.push({ + ...field, + fieldName: field.name, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + stats.aggregatableNotExistsFields.push({ + ...field, + fieldName: field.name, + existsInDocs: false, + stats: undefined, + }); + } + } else { + const fieldData = { + fieldName: field.name, + existsInDocs: true, + }; + if (count > 0) { + stats.nonAggregatableExistsFields.push(fieldData); + } else { + stats.nonAggregatableNotExistsFields.push(fieldData); + } + } + }); + return stats; + } catch (error) { + handleError({ + error, + request, + onError, + title: i18n.translate('xpack.dataVisualizer.esql.countAndCardinalityError', { + defaultMessage: + 'Unable to fetch count & cardinality for {count} {count, plural, one {field} other {fields}}: {fieldNames}', + values: { + count: aggregatableFieldsToQuery.length, + fieldNames: aggregatableFieldsToQuery.map((r) => r.name).join(), + }, + }), + }); + return Promise.reject(error); + } + } +}; + +/** + * Fetching count and cardinality in chunks of 30 fields per request in parallel + * limiting at 10 requests maximum at a time + * @param runRequest + * @param fields + * @param esqlBaseQueryWithLimit + */ +export const getESQLOverallStats = async ({ + runRequest, + fields, + esqlBaseQueryWithLimit, + filter, + limitSize, + totalCount, + onError, +}: { + runRequest: UseCancellableSearch['runRequest']; + fields: Column[]; + esqlBaseQueryWithLimit: string; + filter?: estypes.QueryDslQueryContainer; + limitSize?: ESQLDefaultLimitSizeOption; + totalCount: number; + onError?: HandleErrorCallback; +}) => { + const limiter = pLimit(MAX_CONCURRENT_REQUESTS); + + const chunkedFields = chunk(fields, 30); + + const resp = await Promise.allSettled( + chunkedFields.map((groupedFields, idx) => + limiter(() => + getESQLOverallStatsInChunk({ + runRequest, + fields: groupedFields, + esqlBaseQueryWithLimit, + limitSize, + filter, + totalCount, + onError, + }) + ) + ) + ); + const results = resp.filter(isFulfilled).map((r) => r.value); + + const stats = results.reduce( + (acc, result) => { + if (acc && result) { + acc.aggregatableExistsFields.push(...result.aggregatableExistsFields); + acc.aggregatableNotExistsFields.push(...result.aggregatableNotExistsFields); + acc.nonAggregatableExistsFields.push(...result.nonAggregatableExistsFields); + acc.nonAggregatableNotExistsFields.push(...result.nonAggregatableNotExistsFields); + } + return acc; + }, + { + aggregatableExistsFields: [] as AggregatableField[], + aggregatableNotExistsFields: [] as AggregatableField[], + nonAggregatableExistsFields: [] as NonAggregatableField[], + nonAggregatableNotExistsFields: [] as NonAggregatableField[], + } + ); + + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts new file mode 100644 index 00000000000000..fb068994665769 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import { getSafeESQLName } from '../requests/esql_utils'; +import type { DateFieldStats, FieldStatsError } from '../../../../../common/types/field_stats'; + +interface Params { + runRequest: UseCancellableSearch['runRequest']; + columns: Column[]; + esqlBaseQuery: string; + filter?: QueryDslQueryContainer; +} + +export const getESQLDateFieldStats = async ({ + runRequest, + columns, + esqlBaseQuery, + filter, +}: Params) => { + const dateFields = columns.map((field) => { + return { + field, + query: `${getSafeESQLName(`${field.name}_earliest`)} = MIN(${getSafeESQLName( + field.name + )}), ${getSafeESQLName(`${field.name}_latest`)} = MAX(${getSafeESQLName(field.name)})`, + }; + }); + + if (dateFields.length > 0) { + const dateStatsQuery = ' | STATS ' + dateFields.map(({ query }) => query).join(','); + const request = { + params: { + query: esqlBaseQuery + dateStatsQuery, + ...(filter ? { filter } : {}), + }, + }; + try { + const dateFieldsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + + if (dateFieldsResp) { + return dateFields.map(({ field: dateField }, idx) => { + const row = dateFieldsResp.rawResponse.values[0] as Array; + + const earliest = row[idx * 2]; + const latest = row[idx * 2 + 1]; + + return { + fieldName: dateField.name, + earliest, + latest, + } as DateFieldStats; + }); + } + } catch (error) { + // Log for debugging purposes + // eslint-disable-next-line no-console + console.error(error, request); + return dateFields.map(({ field }, idx) => { + return { + fieldName: field.name, + error, + } as FieldStatsError; + }); + } + } + return []; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts new file mode 100644 index 00000000000000..0ca4e95ac69a8e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import pLimit from 'p-limit'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import { getSafeESQLName } from '../requests/esql_utils'; +import { isFulfilled, isRejected } from '../../../common/util/promise_all_settled_utils'; +import { MAX_CONCURRENT_REQUESTS } from '../../constants/index_data_visualizer_viewer'; +import type { BucketCount, BucketTerm } from '../../types/esql_data_visualizer'; +import type { FieldStatsError, StringFieldStats } from '../../../../../common/types/field_stats'; + +interface Params { + runRequest: UseCancellableSearch['runRequest']; + columns: Column[]; + esqlBaseQuery: string; + filter?: QueryDslQueryContainer; +} +export const getESQLKeywordFieldStats = async ({ + runRequest, + columns, + esqlBaseQuery, + filter, +}: Params) => { + const limiter = pLimit(MAX_CONCURRENT_REQUESTS); + + const keywordFields = columns.map((field) => { + const query = + esqlBaseQuery + + `| STATS ${getSafeESQLName(`${field.name}_terms`)} = count(${getSafeESQLName( + field.name + )}) BY ${getSafeESQLName(field.name)} + | LIMIT 10 + | SORT ${getSafeESQLName(`${field.name}_terms`)} DESC`; + return { + field, + request: { + params: { + query, + ...(filter ? { filter } : {}), + }, + }, + }; + }); + + if (keywordFields.length > 0) { + const keywordTopTermsResp = await Promise.allSettled( + keywordFields.map(({ request }) => + limiter(() => runRequest(request, { strategy: ESQL_SEARCH_STRATEGY })) + ) + ); + if (keywordTopTermsResp) { + return keywordFields.map(({ field, request }, idx) => { + const resp = keywordTopTermsResp[idx]; + if (!resp) return; + + if (isFulfilled(resp)) { + const results = resp.value?.rawResponse.values as Array<[BucketCount, BucketTerm]>; + if (results) { + const topValuesSampleSize = results?.reduce((acc: number, row) => acc + row[0], 0); + + const terms = results.map((row) => ({ + key: row[1], + doc_count: row[0], + percent: row[0] / topValuesSampleSize, + })); + + return { + fieldName: field.name, + topValues: terms, + topValuesSampleSize, + topValuesSamplerShardSize: topValuesSampleSize, + isTopValuesSampled: false, + } as StringFieldStats; + } + return; + } + + if (isRejected(resp)) { + // Log for debugging purposes + // eslint-disable-next-line no-console + console.error(resp, request); + + return { + fieldName: field.name, + error: resp.reason, + } as FieldStatsError; + } + }); + } + } + return []; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts new file mode 100644 index 00000000000000..ca8684499eb3c0 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { chunk } from 'lodash'; +import pLimit from 'p-limit'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import { processDistributionData } from '../../utils/process_distribution_data'; +import { PERCENTILE_SPACING } from '../requests/constants'; +import { getESQLPercentileQueryArray, getSafeESQLName, PERCENTS } from '../requests/esql_utils'; +import { isFulfilled } from '../../../common/util/promise_all_settled_utils'; +import { MAX_CONCURRENT_REQUESTS } from '../../constants/index_data_visualizer_viewer'; +import { handleError } from './handle_error'; +import type { + FieldStatsError, + NonSampledNumericFieldStats, +} from '../../../../../common/types/field_stats'; + +interface Params { + runRequest: UseCancellableSearch['runRequest']; + columns: Column[]; + esqlBaseQuery: string; + filter?: QueryDslQueryContainer; +} +const getESQLNumericFieldStatsInChunk = async ({ + runRequest, + columns, + esqlBaseQuery, + filter, +}: Params): Promise> => { + // Hashmap of agg to index/order of which is made in the ES|QL query + // {min: 0, max: 1, p0: 2, p5: 3, ..., p100: 22} + const numericAccessorMap = PERCENTS.reduce<{ [key: string]: number }>( + (acc, curr, idx) => { + // +2 for the min and max aggs + acc[`p${curr}`] = idx + 2; + return acc; + }, + { + // First two are min and max aggs + min: 0, + max: 1, + // and percentiles p0, p5, ..., p100 are the rest + } + ); + const numericFields = columns.map((field, idx) => { + const percentiles = getESQLPercentileQueryArray(field.name, PERCENTS); + return { + field, + query: `${getSafeESQLName(`${field.name}_min`)} = MIN(${getSafeESQLName(field.name)}), + ${getSafeESQLName(`${field.name}_max`)} = MAX(${getSafeESQLName(field.name)}), + ${percentiles.join(',')} + `, + // Start index of field in the response, so we know to slice & access the values + startIndex: idx * Object.keys(numericAccessorMap).length, + }; + }); + + if (numericFields.length > 0) { + const numericStatsQuery = '| STATS ' + numericFields.map(({ query }) => query).join(','); + + const request = { + params: { + query: esqlBaseQuery + numericStatsQuery, + ...(filter ? { filter } : {}), + }, + }; + try { + const fieldStatsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + + if (fieldStatsResp) { + const values = fieldStatsResp.rawResponse.values[0]; + + return numericFields.map(({ field, startIndex }, idx) => { + /** Order of aggs we are expecting back from query + * 0 = min; 23 = startIndex + 0 for 2nd field + * 1 = max; 24 = startIndex + 1 + * 2 p0; 25; 24 = startIndex + 2 + * 3 p5; 26 + * 4 p10; 27 + * ... + * 22 p100; + */ + const min = values[startIndex + numericAccessorMap.min]; + const max = values[startIndex + numericAccessorMap.max]; + const median = values[startIndex + numericAccessorMap.p50]; + + const percentiles = values + .slice(startIndex + numericAccessorMap.p0, startIndex + numericAccessorMap.p100) + .map((value: number) => ({ value })); + + const distribution = processDistributionData(percentiles, PERCENTILE_SPACING, min); + + return { + fieldName: field.name, + ...field, + min, + max, + median, + distribution, + } as NonSampledNumericFieldStats; + }); + } + } catch (error) { + handleError({ error, request }); + return numericFields.map(({ field }) => { + return { + fieldName: field.name, + error, + } as FieldStatsError; + }); + } + } + return []; +}; + +export const getESQLNumericFieldStats = async ({ + runRequest, + columns, + esqlBaseQuery, + filter, +}: Params): Promise> => { + const limiter = pLimit(MAX_CONCURRENT_REQUESTS); + + // Breakdown so that each requests only contains 10 numeric fields + // to prevent potential circuit breaking exception + // or too big of a payload + const numericColumnChunks = chunk(columns, 10); + const numericStats = await Promise.allSettled( + numericColumnChunks.map((numericColumns) => + limiter(() => + getESQLNumericFieldStatsInChunk({ + columns: numericColumns, + filter, + runRequest, + esqlBaseQuery, + }) + ) + ) + ); + + return numericStats.filter(isFulfilled).flatMap((stat) => stat.value); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts new file mode 100644 index 00000000000000..f4bc710a058391 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; +import type { FieldExamples, FieldStatsError } from '../../../../../common/types/field_stats'; + +interface Params { + runRequest: UseCancellableSearch['runRequest']; + columns: Column[]; + esqlBaseQuery: string; + filter?: QueryDslQueryContainer; +} + +/** + * Make one query that gets the top 10 rows for each text field requested + * then process the values to showcase examples for each field + * @param + * @returns + */ +export const getESQLTextFieldStats = async ({ + runRequest, + columns: textFields, + esqlBaseQuery, + filter, +}: Params): Promise> => { + try { + if (textFields.length > 0) { + const request = { + params: { + query: + esqlBaseQuery + + `| KEEP ${textFields.map((f) => f.name).join(',')} + | LIMIT 10`, + ...(filter ? { filter } : {}), + }, + }; + const textFieldsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + + if (textFieldsResp) { + return textFields.map((textField, idx) => { + const examples = (textFieldsResp.rawResponse.values as unknown[][]).map( + (row) => row[idx] + ); + + return { + fieldName: textField.name, + examples, + } as FieldExamples; + }); + } + } + } catch (error) { + return textFields.map((textField, idx) => ({ + fieldName: textField.name, + error, + })) as FieldStatsError[]; + } + return []; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/handle_error.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/handle_error.ts new file mode 100644 index 00000000000000..0d3f845b13f49a --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/handle_error.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +interface DataVizError extends Error { + handled?: boolean; +} +export type HandleErrorCallback = (e: DataVizError, title?: string) => void; + +export const handleError = ({ + onError, + request, + error, + title, +}: { + error: DataVizError; + request: object; + onError?: HandleErrorCallback; + title?: string; +}) => { + // Log error and request for debugging purposes + // eslint-disable-next-line no-console + console.error(error, request); + if (onError) { + error.handled = true; + error.message = JSON.stringify(request); + onError( + error, + title ?? + i18n.translate('xpack.dataVisualizer.esql.errorMessage', { + defaultMessage: 'Error excecuting ES|QL request:', + }) + ); + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.test.ts new file mode 100644 index 00000000000000..d96346f36909fd --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getESQLPercentileQueryArray } from './esql_utils'; + +describe('getESQLPercentileQueryArray', () => { + test('should return correct ESQL query', () => { + const query = getESQLPercentileQueryArray('@odd_field', [0, 50, 100]); + expect(query).toEqual([ + '`@odd_field_p0` = PERCENTILE(`@odd_field`, 0)', + '`@odd_field_p50` = PERCENTILE(`@odd_field`, 50)', + '`@odd_field_p100` = PERCENTILE(`@odd_field`, 100)', + ]); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts new file mode 100644 index 00000000000000..334b0b06bb0ab5 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { MAX_PERCENT, PERCENTILE_SPACING } from './constants'; + +export interface ESQLQuery { + esql: string; +} + +/** + * Helper function to escape special characters for field names used in ES|QL queries. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-syntax.html#esql-identifiers + * @param str + * @returns "`str`" + **/ +export const getSafeESQLName = (str: string) => { + return `\`${str}\``; +}; + +export function isESQLQuery(arg: unknown): arg is ESQLQuery { + return isPopulatedObject(arg, ['esql']); +} +export const PERCENTS = Array.from( + Array(MAX_PERCENT / PERCENTILE_SPACING + 1), + (_, i) => i * PERCENTILE_SPACING +); + +export const getESQLPercentileQueryArray = (fieldName: string, percents = PERCENTS) => + percents.map( + (p) => + `${getSafeESQLName(`${fieldName}_p${p}`)} = PERCENTILE(${getSafeESQLName(fieldName)}, ${p})` + ); + +export const getSafeESQLLimitSize = (str?: string) => { + if (str === 'none' || !str) return ''; + return ` | LIMIT ${str}`; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_data_view_by_index_pattern.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_data_view_by_index_pattern.ts new file mode 100644 index 00000000000000..8d0385e8d9f9e5 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_data_view_by_index_pattern.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public'; + +/** + * Get a saved data view that matches the index pattern (as close as possible) + * or create a new adhoc data view if no matches found + * @param dataViews + * @param indexPatternFromQuery + * @param currentDataView + * @returns + */ +export async function getOrCreateDataViewByIndexPattern( + dataViews: DataViewsContract, + indexPatternFromQuery: string | undefined, + currentDataView: DataView | undefined +) { + if (indexPatternFromQuery) { + const matched = await dataViews.find(indexPatternFromQuery); + + // Only returns persisted data view if it matches index pattern exactly + // Because * in pattern can result in misleading matches (i.e. "kibana*" will return data view with pattern "kibana_1") + // which is not neccessarily the one we want to use + if (matched.length > 0 && matched[0].getIndexPattern() === indexPatternFromQuery) + return matched[0]; + } + + if ( + indexPatternFromQuery && + (currentDataView?.isPersisted() || indexPatternFromQuery !== currentDataView?.getIndexPattern()) + ) { + const dataViewObj = await dataViews.create({ + title: indexPatternFromQuery, + }); + + if (dataViewObj.fields.getByName('@timestamp')?.type === 'date') { + dataViewObj.timeFieldName = '@timestamp'; + } + return dataViewObj; + } + return currentDataView; +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts index 690b8ec29740e9..2fe7f9489e145a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -25,7 +25,7 @@ export const getDocumentCountStats = async ( search: DataPublicPluginStart['search'], params: OverallStatsSearchStrategyParams, searchOptions: ISearchOptions, - browserSessionSeed: string, + browserSessionSeed?: string, probability?: number | null, minimumRandomSamplerDocCount?: number ): Promise => { @@ -193,6 +193,7 @@ export const getDocumentCountStats = async ( } } } + return result; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index f7d1b39f15d3fe..a084c5bfa36879 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -21,7 +21,7 @@ import { isDefined } from '@kbn/ml-is-defined'; import { extractErrorProperties } from '@kbn/ml-error-utils'; import { processTopValues } from './utils'; import { buildAggregationWithSamplingOption } from './build_random_sampler_agg'; -import { MAX_PERCENT, PERCENTILE_SPACING, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; +import { MAX_PERCENT, PERCENTILE_SPACING } from './constants'; import type { Aggs, Bucket, @@ -154,7 +154,6 @@ export const fetchNumericFieldsStats = ( fields: Field[], options: ISearchOptions ): Observable => { - const { samplerShardSize } = params; const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); return dataSearch @@ -183,9 +182,6 @@ export const fetchNumericFieldsStats = ( ); const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } const fieldAgg = get(aggregations, [...topAggsPath], {}) as { buckets: Bucket[] }; const { topValuesSampleSize, topValues } = processTopValues(fieldAgg); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 159be48b338e42..f3b70085de33fb 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -19,7 +19,6 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { extractErrorProperties } from '@kbn/ml-error-utils'; import { processTopValues } from './utils'; import { buildAggregationWithSamplingOption } from './build_random_sampler_agg'; -import { SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; import type { Aggs, Field, @@ -71,7 +70,6 @@ export const fetchStringFieldsStats = ( fields: Field[], options: ISearchOptions ): Observable => { - const { samplerShardSize } = params; const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); return dataSearch @@ -94,9 +92,6 @@ export const fetchStringFieldsStats = ( const safeFieldName = field.safeFieldName; const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } const fieldAgg = get(aggregations, [...topAggsPath], {}); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/esql_data_visualizer.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/esql_data_visualizer.ts new file mode 100644 index 00000000000000..d4c4db162d97a6 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/esql_data_visualizer.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type BucketCount = number; +export type BucketTerm = string; +export interface AggregatableField { + fieldName: string; + existsInDocs: boolean; + stats?: { + sampleCount: number; + count: number; + cardinality: number; + }; + aggregatable?: boolean; +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts index 508d6e0015446e..960e3eb2695472 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts @@ -31,7 +31,6 @@ export interface DataVisualizerIndexBasedAppState extends Omit | null; export type DVKey = keyof Exclude; @@ -32,6 +35,8 @@ export type DVStorageMapped = T extends typeof DV_FROZEN_TIER_P ? number | null : T extends typeof DV_DATA_DRIFT_DISTRIBUTION_CHART_TYPE ? DATA_DRIFT_COMPARISON_CHART_TYPE + : T extends typeof DV_ESQL_LIMIT_SIZE + ? ESQLDefaultLimitSizeOption : null; export const DV_STORAGE_KEYS = [ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/get_supported_aggs.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/get_supported_aggs.ts index 26c71a7101c7fd..8f4f1c570b94e6 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/get_supported_aggs.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/get_supported_aggs.ts @@ -74,3 +74,11 @@ export const getSupportedAggs = (field: DataViewField) => { if (field.aggregatable) return SUPPORTED_AGGS.AGGREGATABLE; return SUPPORTED_AGGS.DEFAULT; }; + +export const getESQLSupportedAggs = ( + field: { name: string; type: string }, + aggregatable = true +) => { + if (aggregatable) return SUPPORTED_AGGS.AGGREGATABLE; + return SUPPORTED_AGGS.DEFAULT; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts index 46719c06e22643..3fea814b32d7e4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { isDefined } from '@kbn/ml-is-defined'; import { last } from 'lodash'; import type { Distribution } from '../../../../common/types/field_stats'; @@ -70,6 +71,8 @@ export const processDistributionData = ( for (let i = 0; i < totalBuckets; i++) { const bucket = percentileBuckets[i]; + if (!isDefined(bucket.value)) continue; + // Results from the percentiles aggregation can have precision rounding // artifacts e.g returning 200 and 200.000000000123, so check for equality // around double floating point precision i.e. 15 sig figs. diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json index 9f22cf2e8bda07..9f30b3db0eb8d8 100644 --- a/x-pack/plugins/data_visualizer/tsconfig.json +++ b/x-pack/plugins/data_visualizer/tsconfig.json @@ -40,6 +40,7 @@ "@kbn/lens-plugin", "@kbn/maps-plugin", "@kbn/ml-agg-utils", + "@kbn/ml-cancellable-search", "@kbn/ml-date-picker", "@kbn/ml-is-defined", "@kbn/ml-is-populated-object", @@ -70,7 +71,9 @@ "@kbn/ml-chi2test", "@kbn/field-utils", "@kbn/visualization-utils", + "@kbn/text-based-languages", "@kbn/code-editor", + "@kbn/es-types", "@kbn/ui-theme" ], "exclude": [ diff --git a/x-pack/plugins/ml/common/constants/locator.ts b/x-pack/plugins/ml/common/constants/locator.ts index 614c037c13026c..cc07b7b4f27a1f 100644 --- a/x-pack/plugins/ml/common/constants/locator.ts +++ b/x-pack/plugins/ml/common/constants/locator.ts @@ -40,6 +40,7 @@ export const ML_PAGES = { * Page: Data Visualizer * Open index data visualizer viewer page */ + DATA_VISUALIZER_ESQL: 'datavisualizer/esql', DATA_VISUALIZER_INDEX_VIEWER: 'jobs/new_job/datavisualizer', ANOMALY_DETECTION_CREATE_JOB: 'jobs/new_job', ANOMALY_DETECTION_CREATE_JOB_RECOGNIZER: 'jobs/new_job/recognize', diff --git a/x-pack/plugins/ml/common/types/locator.ts b/x-pack/plugins/ml/common/types/locator.ts index 85b2550eb8e307..10b6122910b71e 100644 --- a/x-pack/plugins/ml/common/types/locator.ts +++ b/x-pack/plugins/ml/common/types/locator.ts @@ -44,6 +44,7 @@ export interface MlGenericUrlPageState extends MlIndexBasedSearchState { export type MlGenericUrlState = MLPageState< | typeof ML_PAGES.DATA_VISUALIZER_INDEX_VIEWER + | typeof ML_PAGES.DATA_VISUALIZER_ESQL | typeof ML_PAGES.ANOMALY_DETECTION_CREATE_JOB | typeof ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_RECOGNIZER | typeof ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_ADVANCED diff --git a/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx b/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx index dda32fbaf8af76..e2e5c115cd06a9 100644 --- a/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx +++ b/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx @@ -234,6 +234,16 @@ export function useSideNavItems(activeRoute: MlRoute | undefined) { disabled: false, testSubj: 'mlMainTab indexDataVisualizer', }, + { + id: 'esql_datavisualizer', + pathId: ML_PAGES.DATA_VISUALIZER_ESQL, + name: i18n.translate('xpack.ml.navMenu.esqlDataVisualizerLinkText', { + defaultMessage: 'ES|QL', + }), + disabled: false, + testSubj: 'mlMainTab esqlDataVisualizer', + }, + { id: 'data_drift', pathId: ML_PAGES.DATA_DRIFT_INDEX_SELECT, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx index 0a3f49cc882ff4..153978c56d7e5a 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx @@ -17,6 +17,8 @@ import { EuiLink, EuiSpacer, EuiText, + EuiBetaBadge, + EuiTextAlign, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -25,6 +27,7 @@ import { isFullLicense } from '../license'; import { useMlKibana, useNavigateToPath } from '../contexts/kibana'; import { HelpMenu } from '../components/help_menu'; import { MlPageHeader } from '../components/page_header'; +import { ML_PAGES } from '../../locator'; function startTrialDescription() { return ( @@ -49,6 +52,7 @@ function startTrialDescription() { export const DatavisualizerSelector: FC = () => { useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); + const { services: { licenseManagement, @@ -154,6 +158,54 @@ export const DatavisualizerSelector: FC = () => { data-test-subj="mlDataVisualizerCardIndexData" /> + + } + title={ + + <> + {' '} + + } + tooltipPosition={'right'} + /> + + + } + description={ + + } + footer={ + navigateToPath(ML_PAGES.DATA_VISUALIZER_ESQL)} + data-test-subj="mlDataVisualizerSelectESQLButton" + > + + + } + data-test-subj="mlDataVisualizerCardESQLData" + /> + {startTrialVisible === true && ( diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx index 69c034738cf5f2..092e65f69c35dc 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx @@ -15,6 +15,7 @@ import type { GetAdditionalLinksParams, } from '@kbn/data-visualizer-plugin/public'; import { useTimefilter } from '@kbn/ml-date-picker'; +import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui'; import { useMlKibana, useMlLocator } from '../../contexts/kibana'; import { HelpMenu } from '../../components/help_menu'; import { ML_PAGES } from '../../../../common/constants/locator'; @@ -23,8 +24,9 @@ import { mlNodesAvailable, getMlNodeCount } from '../../ml_nodes_check/check_ml_ import { checkPermission } from '../../capabilities/check_capabilities'; import { MlPageHeader } from '../../components/page_header'; import { useEnabledFeatures } from '../../contexts/ml'; +import { TechnicalPreviewBadge } from '../../components/technical_preview_badge/technical_preview_badge'; -export const IndexDataVisualizerPage: FC = () => { +export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false }) => { useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); const { services: { @@ -180,19 +182,34 @@ export const IndexDataVisualizerPage: FC = () => { // eslint-disable-next-line react-hooks/exhaustive-deps [mlLocator, mlFeaturesDisabled] ); + const { euiTheme } = useEuiTheme(); + return IndexDataVisualizer ? ( {IndexDataVisualizer !== null ? ( <> - + + + {esql ? ( + <> + + + + + + + + ) : null} + ) : null} diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/page.tsx index 0a53ac7b4a81e8..d88f0e5cd2674e 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/page.tsx @@ -6,7 +6,7 @@ */ import React, { FC, useCallback } from 'react'; -import { EuiPageBody, EuiPanel } from '@elastic/eui'; +import { EuiFlexGroup, EuiPageBody, EuiPanel } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { SavedObjectFinder } from '@kbn/saved-objects-finder-plugin/public'; @@ -20,7 +20,13 @@ export interface PageProps { const RESULTS_PER_PAGE = 20; -export const Page: FC = ({ nextStepPath }) => { +export const Page: FC = ({ + nextStepPath, + extraButtons, +}: { + nextStepPath: string; + extraButtons?: React.ReactNode; +}) => { const { contentManagement, uiSettings } = useMlKibana().services; const navigateToPath = useNavigateToPath(); @@ -80,7 +86,13 @@ export const Page: FC = ({ nextStepPath }) => { uiSettings, }} > - + + + {extraButtons ? extraButtons : null} + diff --git a/x-pack/plugins/ml/public/application/routing/components/navigate_to_page_button.tsx b/x-pack/plugins/ml/public/application/routing/components/navigate_to_page_button.tsx new file mode 100644 index 00000000000000..fcc7a01e9f508a --- /dev/null +++ b/x-pack/plugins/ml/public/application/routing/components/navigate_to_page_button.tsx @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { EuiButton } from '@elastic/eui'; +import { useNavigateToPath } from '../../contexts/kibana'; + +export const NavigateToPageButton = ({ + nextStepPath, + title, +}: { + nextStepPath: string; + title: string | React.ReactNode; +}) => { + const navigateToPath = useNavigateToPath(); + const onClick = useCallback(() => { + navigateToPath(nextStepPath); + }, [navigateToPath, nextStepPath]); + + return {title}; +}; diff --git a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx index f14eff64675fc1..e0c327a23c34e9 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx @@ -24,7 +24,7 @@ export const indexBasedRouteFactory = ( title: i18n.translate('xpack.ml.dataVisualizer.dataView.docTitle', { defaultMessage: 'Index Data Visualizer', }), - render: () => , + render: () => , breadcrumbs: [ getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath), getBreadcrumbWithUrlForApp('DATA_VISUALIZER_BREADCRUMB', navigateToPath, basePath), @@ -36,13 +36,34 @@ export const indexBasedRouteFactory = ( ], }); -const PageWrapper: FC = () => { +export const indexESQLBasedRouteFactory = ( + navigateToPath: NavigateToPath, + basePath: string +): MlRoute => ({ + id: 'data_view_datavisualizer_esql', + path: createPath(ML_PAGES.DATA_VISUALIZER_ESQL), + title: i18n.translate('xpack.ml.dataVisualizer.esql.docTitle', { + defaultMessage: 'Index Data Visualizer (ES|QL)', + }), + render: () => , + breadcrumbs: [ + getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath), + getBreadcrumbWithUrlForApp('DATA_VISUALIZER_BREADCRUMB', navigateToPath, basePath), + { + text: i18n.translate('xpack.ml.dataFrameAnalyticsBreadcrumbs.esqlLabel', { + defaultMessage: 'Index Data Visualizer (ES|QL)', + }), + }, + ], +}); + +const PageWrapper: FC<{ esql: boolean }> = ({ esql }) => { const { context } = useRouteResolver('basic', []); return ( - + ); diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 24976cffc43d9c..b790490cc5e9f4 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -8,6 +8,7 @@ import React, { FC } from 'react'; import { Redirect } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import { ML_PAGES } from '../../../../locator'; import { NavigateToPath, useMlKibana } from '../../../contexts/kibana'; import { createPath, MlRoute, PageLoader, PageProps } from '../../router'; @@ -15,6 +16,7 @@ import { useRouteResolver } from '../../use_resolver'; import { basicResolvers } from '../../resolvers'; import { Page, preConfiguredJobRedirect } from '../../../jobs/new_job/pages/index_or_search'; import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs'; +import { NavigateToPageButton } from '../../components/navigate_to_page_button'; enum MODE { NEW_JOB, @@ -24,6 +26,7 @@ enum MODE { interface IndexOrSearchPageProps extends PageProps { nextStepPath: string; mode: MODE; + extraButtons?: React.ReactNode; } const getBreadcrumbs = (navigateToPath: NavigateToPath, basePath: string) => [ @@ -104,14 +107,28 @@ export const dataVizIndexOrSearchRouteFactory = ( title: i18n.translate('xpack.ml.selectDataViewLabel', { defaultMessage: 'Select Data View', }), - render: (props, deps) => ( - - ), + render: (props, deps) => { + const button = ( + + } + /> + ); + return ( + + ); + }, breadcrumbs: getDataVisBreadcrumbs(navigateToPath, basePath), }); @@ -185,7 +202,7 @@ export const changePointDetectionIndexOrSearchRouteFactory = ( breadcrumbs: getChangePointDetectionBreadcrumbs(navigateToPath, basePath), }); -const PageWrapper: FC = ({ nextStepPath, mode }) => { +const PageWrapper: FC = ({ nextStepPath, mode, extraButtons }) => { const { services: { http: { basePath }, @@ -207,7 +224,7 @@ const PageWrapper: FC = ({ nextStepPath, mode }) => { ); return ( - + ); }; diff --git a/x-pack/plugins/ml/public/locator/ml_locator.ts b/x-pack/plugins/ml/public/locator/ml_locator.ts index 05fe312fd9a46b..1034101d3211e7 100644 --- a/x-pack/plugins/ml/public/locator/ml_locator.ts +++ b/x-pack/plugins/ml/public/locator/ml_locator.ts @@ -88,6 +88,7 @@ export class MlLocatorDefinition implements LocatorDefinition { case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_FROM_PATTERN_ANALYSIS: case ML_PAGES.DATA_VISUALIZER: case ML_PAGES.DATA_VISUALIZER_FILE: + case ML_PAGES.DATA_VISUALIZER_ESQL: case ML_PAGES.DATA_VISUALIZER_INDEX_VIEWER: case ML_PAGES.DATA_VISUALIZER_INDEX_SELECT: case ML_PAGES.AIOPS: diff --git a/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts b/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts index 539c42bf763ca7..a66f2fca4e2c17 100644 --- a/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts +++ b/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts @@ -263,6 +263,17 @@ function createDeepLinks( }; }, + getESQLDataVisualizerDeepLink: (): AppDeepLink => { + return { + id: 'indexDataVisualizer', + title: i18n.translate('xpack.ml.deepLink.esqlDataVisualizer', { + defaultMessage: 'ES|QL Data Visualizer', + }), + path: `/${ML_PAGES.DATA_VISUALIZER_ESQL}`, + navLinkStatus: getNavStatus(true), + }; + }, + getDataDriftDeepLink: (): AppDeepLink => { return { id: 'dataDrift', diff --git a/yarn.lock b/yarn.lock index 754fe46ae1ea73..ecbb61b69b282f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5172,6 +5172,10 @@ version "0.0.0" uid "" +"@kbn/ml-cancellable-search@link:x-pack/packages/ml/cancellable_search": + version "0.0.0" + uid "" + "@kbn/ml-category-validator@link:x-pack/packages/ml/category_validator": version "0.0.0" uid "" From 9a72723d71e24ebda1a866922470ffebdd8aa172 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Wed, 31 Jan 2024 14:15:34 -0500 Subject: [PATCH 14/20] [RAM] SRE Alert feedbacks (#175939) ## Summary - [x] In rule details page default tabs to the Alert tabs instead of the execution history - [x] Default alert search bar to see the last 24 hours alerts --- .../containers/state_container.tsx | 2 +- .../get_rule_details_table_configuration.tsx | 4 +-- .../components/rule_details_tabs.tsx | 28 +++++++++---------- .../pages/rule_details/rule_details.tsx | 2 +- .../pages/alerts/state_synchronization.ts | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/observability/public/components/alert_search_bar/containers/state_container.tsx b/x-pack/plugins/observability/public/components/alert_search_bar/containers/state_container.tsx index 5513cc62df6263..a08971b0c350e3 100644 --- a/x-pack/plugins/observability/public/components/alert_search_bar/containers/state_container.tsx +++ b/x-pack/plugins/observability/public/components/alert_search_bar/containers/state_container.tsx @@ -35,7 +35,7 @@ interface AlertSearchBarStateTransitions { } const defaultState: AlertSearchBarContainerState = { - rangeFrom: 'now-2h', + rangeFrom: 'now-24h', rangeTo: 'now', kuery: '', status: ALL_ALERTS.status, diff --git a/x-pack/plugins/observability/public/components/alerts_table/rule_details/get_rule_details_table_configuration.tsx b/x-pack/plugins/observability/public/components/alerts_table/rule_details/get_rule_details_table_configuration.tsx index 369a3d4c5c12d4..d03e90452fa28e 100644 --- a/x-pack/plugins/observability/public/components/alerts_table/rule_details/get_rule_details_table_configuration.tsx +++ b/x-pack/plugins/observability/public/components/alerts_table/rule_details/get_rule_details_table_configuration.tsx @@ -6,7 +6,7 @@ */ import React from 'react'; -import { TIMESTAMP } from '@kbn/rule-data-utils'; +import { ALERT_START } from '@kbn/rule-data-utils'; import { SortOrder } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { AlertsTableConfigurationRegistry, @@ -35,7 +35,7 @@ export const getRuleDetailsTableConfiguration = ( }), sort: [ { - [TIMESTAMP]: { + [ALERT_START]: { order: 'desc' as SortOrder, }, }, diff --git a/x-pack/plugins/observability/public/pages/rule_details/components/rule_details_tabs.tsx b/x-pack/plugins/observability/public/pages/rule_details/components/rule_details_tabs.tsx index 163aee618f9b9d..f5e514d57a68a7 100644 --- a/x-pack/plugins/observability/public/pages/rule_details/components/rule_details_tabs.tsx +++ b/x-pack/plugins/observability/public/pages/rule_details/components/rule_details_tabs.tsx @@ -68,20 +68,6 @@ export function RuleDetailsTabs({ ]); const tabs: EuiTabbedContentTab[] = [ - { - id: RULE_DETAILS_EXECUTION_TAB, - name: i18n.translate('xpack.observability.ruleDetails.rule.eventLogTabText', { - defaultMessage: 'Execution history', - }), - 'data-test-subj': 'eventLogListTab', - content: ( - - - {rule && ruleType ? : null} - - - ), - }, { id: RULE_DETAILS_ALERTS_TAB, name: i18n.translate('xpack.observability.ruleDetails.rule.alertsTabText', { @@ -117,6 +103,20 @@ export function RuleDetailsTabs({ ), }, + { + id: RULE_DETAILS_EXECUTION_TAB, + name: i18n.translate('xpack.observability.ruleDetails.rule.eventLogTabText', { + defaultMessage: 'Execution history', + }), + 'data-test-subj': 'eventLogListTab', + content: ( + + + {rule && ruleType ? : null} + + + ), + }, ]; const handleTabIdChange = (newTabId: TabId) => { diff --git a/x-pack/plugins/observability/public/pages/rule_details/rule_details.tsx b/x-pack/plugins/observability/public/pages/rule_details/rule_details.tsx index 6a9c38b1b1e803..90ed7866a15600 100644 --- a/x-pack/plugins/observability/public/pages/rule_details/rule_details.tsx +++ b/x-pack/plugins/observability/public/pages/rule_details/rule_details.tsx @@ -103,7 +103,7 @@ export function RuleDetailsPage() { return urlTabId && [RULE_DETAILS_EXECUTION_TAB, RULE_DETAILS_ALERTS_TAB].includes(urlTabId) ? (urlTabId as TabId) - : RULE_DETAILS_EXECUTION_TAB; + : RULE_DETAILS_ALERTS_TAB; }); const [esQuery, setEsQuery] = useState<{ bool: BoolQuery }>(); diff --git a/x-pack/test/observability_functional/apps/observability/pages/alerts/state_synchronization.ts b/x-pack/test/observability_functional/apps/observability/pages/alerts/state_synchronization.ts index 9711443fcbf84f..5ea77239f0f060 100644 --- a/x-pack/test/observability_functional/apps/observability/pages/alerts/state_synchronization.ts +++ b/x-pack/test/observability_functional/apps/observability/pages/alerts/state_synchronization.ts @@ -61,7 +61,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await assertAlertsPageState({ kuery: '', // workflowStatus: 'Open', - timeRange: 'Last 2 hours', + timeRange: 'Last 24 hours', }); }); From ff60eb14f8f858d6edc1038c066ba52232b9c1b2 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 31 Jan 2024 19:30:16 +0000 Subject: [PATCH 15/20] skip flaky suite (#175955) --- .../e2e/investigations/timeline_templates/creation.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/creation.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/creation.cy.ts index 0a2f2752633073..2115238da25110 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/creation.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/creation.cy.ts @@ -46,7 +46,8 @@ import { waitForTimelinesPanelToBeLoaded } from '../../../tasks/timelines'; import { TIMELINES_URL } from '../../../urls/navigation'; -describe('Timeline Templates', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/175955 +describe.skip('Timeline Templates', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); deleteTimelines(); From f8f691359220ffee66cecd2561d51835deadfa2b Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 31 Jan 2024 13:32:39 -0600 Subject: [PATCH 16/20] Revert "[Security Solution] [Elastic AI Assistant] Adds internal `Get Evaluate` API and migrates `Post Evaluate` API to OAS (#175338)" This reverts commit 38f0a7aa46199beb1e8aa41b68b8e604f18fc554. --- .../evaluation/get_evaluate_route.gen.ts | 22 - .../evaluation/get_evaluate_route.schema.yaml | 40 -- .../evaluation/post_evaluate_route.gen.ts | 85 ---- .../post_evaluate_route.schema.yaml | 126 ------ .../impl/schemas/index.ts | 26 -- .../kbn-elastic-assistant-common/index.ts | 3 +- .../impl/assistant/api.test.tsx | 49 +++ .../impl/assistant/api.tsx | 59 +++ .../api/capabilities/capabilities.test.tsx | 7 +- .../api/capabilities/capabilities.tsx | 9 +- .../capabilities/use_capabilities.test.tsx | 15 +- .../assistant/api/evaluate/evaluate.test.tsx | 69 --- .../impl/assistant/api/evaluate/evaluate.tsx | 95 ---- .../api/evaluate/use_evaluation_data.tsx | 50 --- .../evaluation_settings.tsx | 23 +- .../use_perform_evaluation.test.tsx | 25 +- .../use_perform_evaluation.tsx | 2 +- .../server/__mocks__/request.ts | 10 +- .../server/lib/langchain/executors/index.ts | 19 - .../server/lib/model_evaluator/evaluation.ts | 3 +- .../elastic_assistant/server/plugin.ts | 2 - .../capabilities/get_capabilities_route.ts | 19 +- .../server/routes/evaluate/get_evaluate.ts | 72 ---- .../routes/evaluate/post_evaluate.test.ts | 12 +- .../server/routes/evaluate/post_evaluate.ts | 408 +++++++++--------- .../server/routes/evaluate/utils.ts | 2 +- .../server/schemas/common.ts | 13 - .../server/schemas/evaluate/post_evaluate.ts | 58 +++ .../plugins/elastic_assistant/tsconfig.json | 1 - 29 files changed, 424 insertions(+), 900 deletions(-) delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts delete mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx delete mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx delete mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx rename x-pack/packages/kbn-elastic-assistant/impl/assistant/{api/evaluate => settings/evaluation_settings}/use_perform_evaluation.test.tsx (81%) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/{api/evaluate => settings/evaluation_settings}/use_perform_evaluation.tsx (97%) delete mode 100644 x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts deleted file mode 100644 index 0a6281d69d1096..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from 'zod'; - -/* - * NOTICE: Do not edit this file manually. - * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. - * - * info: - * title: Get Evaluate API endpoint - * version: 1 - */ - -export type GetEvaluateResponse = z.infer; -export const GetEvaluateResponse = z.object({ - agentExecutors: z.array(z.string()), -}); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml deleted file mode 100644 index b0c0c218eb9ac3..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml +++ /dev/null @@ -1,40 +0,0 @@ -openapi: 3.0.0 -info: - title: Get Evaluate API endpoint - version: '1' -paths: - /internal/elastic_assistant/evaluate: - get: - operationId: GetEvaluate - x-codegen-enabled: true - description: Get relevant data for performing an evaluation like available sample data, agents, and evaluators - summary: Get relevant data for performing an evaluation - tags: - - Evaluation API - responses: - '200': - description: Successful response - content: - application/json: - schema: - type: object - properties: - agentExecutors: - type: array - items: - type: string - required: - - agentExecutors - '400': - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts deleted file mode 100644 index d5d1177a9c16e1..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from 'zod'; - -/* - * NOTICE: Do not edit this file manually. - * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. - * - * info: - * title: Post Evaluate API endpoint - * version: 1 - */ - -export type OutputIndex = z.infer; -export const OutputIndex = z.string().regex(/^.kibana-elastic-ai-assistant-/); - -export type DatasetItem = z.infer; -export const DatasetItem = z.object({ - id: z.string().optional(), - input: z.string(), - prediction: z.string().optional(), - reference: z.string(), - tags: z.array(z.string()).optional(), -}); - -export type Dataset = z.infer; -export const Dataset = z.array(DatasetItem).default([]); - -export type PostEvaluateBody = z.infer; -export const PostEvaluateBody = z.object({ - dataset: Dataset.optional(), - evalPrompt: z.string().optional(), -}); - -export type PostEvaluateRequestQuery = z.infer; -export const PostEvaluateRequestQuery = z.object({ - /** - * Agents parameter description - */ - agents: z.string(), - /** - * Dataset Name parameter description - */ - datasetName: z.string().optional(), - /** - * Evaluation Type parameter description - */ - evaluationType: z.string().optional(), - /** - * Eval Model parameter description - */ - evalModel: z.string().optional(), - /** - * Models parameter description - */ - models: z.string(), - /** - * Output Index parameter description - */ - outputIndex: OutputIndex, - /** - * Project Name parameter description - */ - projectName: z.string().optional(), - /** - * Run Name parameter description - */ - runName: z.string().optional(), -}); -export type PostEvaluateRequestQueryInput = z.input; - -export type PostEvaluateRequestBody = z.infer; -export const PostEvaluateRequestBody = PostEvaluateBody; -export type PostEvaluateRequestBodyInput = z.input; - -export type PostEvaluateResponse = z.infer; -export const PostEvaluateResponse = z.object({ - evaluationId: z.string(), - success: z.boolean(), -}); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml deleted file mode 100644 index 41a7230e85ac5a..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml +++ /dev/null @@ -1,126 +0,0 @@ -openapi: 3.0.0 -info: - title: Post Evaluate API endpoint - version: '1' -paths: - /internal/elastic_assistant/evaluate: - post: - operationId: PostEvaluate - x-codegen-enabled: true - description: Perform an evaluation using sample data against a combination of Agents and Connectors - summary: Performs an evaluation of the Elastic Assistant - tags: - - Evaluation API - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/PostEvaluateBody' - parameters: - - name: agents - in: query - description: Agents parameter description - required: true - schema: - type: string - - name: datasetName - in: query - description: Dataset Name parameter description - schema: - type: string - - name: evaluationType - in: query - description: Evaluation Type parameter description - schema: - type: string - - name: evalModel - in: query - description: Eval Model parameter description - schema: - type: string - - name: models - in: query - description: Models parameter description - required: true - schema: - type: string - - name: outputIndex - in: query - description: Output Index parameter description - required: true - schema: - $ref: '#/components/schemas/OutputIndex' - - name: projectName - in: query - description: Project Name parameter description - schema: - type: string - - name: runName - in: query - description: Run Name parameter description - schema: - type: string - responses: - '200': - description: Successful response - content: - application/json: - schema: - type: object - properties: - evaluationId: - type: string - success: - type: boolean - required: - - evaluationId - - success - '400': - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string -components: - schemas: - OutputIndex: - type: string - pattern: '^.kibana-elastic-ai-assistant-' - DatasetItem: - type: object - properties: - id: - type: string - input: - type: string - prediction: - type: string - reference: - type: string - tags: - type: array - items: - type: string - required: - - input - - reference - Dataset: - type: array - items: - $ref: '#/components/schemas/DatasetItem' - default: [] - PostEvaluateBody: - type: object - properties: - dataset: - $ref: '#/components/schemas/Dataset' - evalPrompt: - type: string diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts deleted file mode 100644 index 4257cb9bae149b..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// API versioning constants -export const API_VERSIONS = { - public: { - v1: '2023-10-31', - }, - internal: { - v1: '1', - }, -}; - -export const PUBLIC_API_ACCESS = 'public'; -export const INTERNAL_API_ACCESS = 'internal'; - -// Evaluation Schemas -export * from './evaluation/post_evaluate_route.gen'; -export * from './evaluation/get_evaluate_route.gen'; - -// Capabilities Schemas -export * from './capabilities/get_capabilities_route.gen'; diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts index e285be395c71cd..a2576038c6f51a 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts @@ -5,8 +5,7 @@ * 2.0. */ -// Schema constants -export * from './impl/schemas'; +export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen'; export { defaultAssistantFeatures } from './impl/capabilities'; export type { AssistantFeatures } from './impl/capabilities'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx index 26a37e12c4e539..4c71c1e63f8b3e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx @@ -13,6 +13,7 @@ import { fetchConnectorExecuteAction, FetchConnectorExecuteAction, getKnowledgeBaseStatus, + postEvaluation, postKnowledgeBase, } from './api'; import type { Conversation, Message } from '../assistant_context/types'; @@ -339,4 +340,52 @@ describe('API tests', () => { await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); }); }); + + describe('postEvaluation', () => { + it('calls the knowledge base API when correct resource path', async () => { + (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true }); + const testProps = { + http: mockHttp, + evalParams: { + agents: ['not', 'alphabetical'], + dataset: '{}', + datasetName: 'Test Dataset', + projectName: 'Test Project Name', + runName: 'Test Run Name', + evalModel: ['not', 'alphabetical'], + evalPrompt: 'evalPrompt', + evaluationType: ['not', 'alphabetical'], + models: ['not', 'alphabetical'], + outputIndex: 'outputIndex', + }, + }; + + await postEvaluation(testProps); + + expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + method: 'POST', + body: '{"dataset":{},"evalPrompt":"evalPrompt"}', + headers: { 'Content-Type': 'application/json' }, + query: { + models: 'alphabetical,not', + agents: 'alphabetical,not', + datasetName: 'Test Dataset', + evaluationType: 'alphabetical,not', + evalModel: 'alphabetical,not', + outputIndex: 'outputIndex', + projectName: 'Test Project Name', + runName: 'Test Run Name', + }, + signal: undefined, + }); + }); + it('returns error when error is an error', async () => { + const error = 'simulated error'; + (mockHttp.fetch as jest.Mock).mockImplementation(() => { + throw new Error(error); + }); + + await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index c18193c7fa0a6a..f04b99c4e46e19 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -16,6 +16,7 @@ import { getOptionalRequestParams, hasParsableResponse, } from './helpers'; +import { PerformEvaluationParams } from './settings/evaluation_settings/use_perform_evaluation'; export interface FetchConnectorExecuteAction { isEnabledRAGAlerts: boolean; @@ -334,3 +335,61 @@ export const deleteKnowledgeBase = async ({ return error as IHttpFetchError; } }; + +export interface PostEvaluationParams { + http: HttpSetup; + evalParams?: PerformEvaluationParams; + signal?: AbortSignal | undefined; +} + +export interface PostEvaluationResponse { + evaluationId: string; + success: boolean; +} + +/** + * API call for evaluating models. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.evalParams] - Params necessary for evaluation + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const postEvaluation = async ({ + http, + evalParams, + signal, +}: PostEvaluationParams): Promise => { + try { + const path = `/internal/elastic_assistant/evaluate`; + const query = { + agents: evalParams?.agents.sort()?.join(','), + datasetName: evalParams?.datasetName, + evaluationType: evalParams?.evaluationType.sort()?.join(','), + evalModel: evalParams?.evalModel.sort()?.join(','), + outputIndex: evalParams?.outputIndex, + models: evalParams?.models.sort()?.join(','), + projectName: evalParams?.projectName, + runName: evalParams?.runName, + }; + + const response = await http.fetch(path, { + method: 'POST', + body: JSON.stringify({ + dataset: JSON.parse(evalParams?.dataset ?? '[]'), + evalPrompt: evalParams?.evalPrompt ?? '', + }), + headers: { + 'Content-Type': 'application/json', + }, + query, + signal, + }); + + return response as PostEvaluationResponse; + } catch (error) { + return error as IHttpFetchError; + } +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx index 30c113eb0e8036..b41d7ac1445549 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx @@ -13,7 +13,7 @@ import { API_ERROR } from '../../translations'; jest.mock('@kbn/core-http-browser'); const mockHttp = { - get: jest.fn(), + fetch: jest.fn(), } as unknown as HttpSetup; describe('Capabilities API tests', () => { @@ -25,14 +25,15 @@ describe('Capabilities API tests', () => { it('calls the internal assistant API for fetching assistant capabilities', async () => { await getCapabilities({ http: mockHttp }); - expect(mockHttp.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { + expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { + method: 'GET', signal: undefined, version: '1', }); }); it('returns API_ERROR when the response status is error', async () => { - (mockHttp.get as jest.Mock).mockResolvedValue({ status: API_ERROR }); + (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: API_ERROR }); const result = await getCapabilities({ http: mockHttp }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx index 96e6660f6bc0ef..59927dbf2c4721 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx @@ -6,7 +6,7 @@ */ import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; -import { API_VERSIONS, GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; +import { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; export interface GetCapabilitiesParams { http: HttpSetup; @@ -29,10 +29,13 @@ export const getCapabilities = async ({ try { const path = `/internal/elastic_assistant/capabilities`; - return await http.get(path, { + const response = await http.fetch(path, { + method: 'GET', signal, - version: API_VERSIONS.internal.v1, + version: '1', }); + + return response as GetCapabilitiesResponse; } catch (error) { return error as IHttpFetchError; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx index b7648983e6f7ad..c9e60b806d1bf3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx @@ -11,12 +11,11 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import type { ReactNode } from 'react'; import React from 'react'; import { useCapabilities, UseCapabilitiesParams } from './use_capabilities'; -import { API_VERSIONS } from '@kbn/elastic-assistant-common'; const statusResponse = { assistantModelEvaluation: true, assistantStreamingEnabled: false }; const http = { - get: jest.fn().mockResolvedValue(statusResponse), + fetch: jest.fn().mockResolvedValue(statusResponse), }; const toasts = { addError: jest.fn(), @@ -37,10 +36,14 @@ describe('useFetchRelatedCases', () => { wrapper: createWrapper(), }); - expect(defaultProps.http.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', { - version: API_VERSIONS.internal.v1, - signal: new AbortController().signal, - }); + expect(defaultProps.http.fetch).toHaveBeenCalledWith( + '/internal/elastic_assistant/capabilities', + { + method: 'GET', + version: '1', + signal: new AbortController().signal, + } + ); expect(toasts.addError).not.toHaveBeenCalled(); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx deleted file mode 100644 index d25953370e97ad..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { postEvaluation } from './evaluate'; -import { HttpSetup } from '@kbn/core-http-browser'; -import { API_VERSIONS } from '@kbn/elastic-assistant-common'; - -jest.mock('@kbn/core-http-browser'); - -const mockHttp = { - post: jest.fn(), -} as unknown as HttpSetup; - -describe('postEvaluation', () => { - it('calls the knowledge base API when correct resource path', async () => { - (mockHttp.post as jest.Mock).mockResolvedValue({ success: true }); - const testProps = { - http: mockHttp, - evalParams: { - agents: ['not', 'alphabetical'], - dataset: '{}', - datasetName: 'Test Dataset', - projectName: 'Test Project Name', - runName: 'Test Run Name', - evalModel: ['not', 'alphabetical'], - evalPrompt: 'evalPrompt', - evaluationType: ['not', 'alphabetical'], - models: ['not', 'alphabetical'], - outputIndex: 'outputIndex', - }, - }; - - await postEvaluation(testProps); - - expect(mockHttp.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { - body: '{"dataset":{},"evalPrompt":"evalPrompt"}', - headers: { 'Content-Type': 'application/json' }, - query: { - models: 'alphabetical,not', - agents: 'alphabetical,not', - datasetName: 'Test Dataset', - evaluationType: 'alphabetical,not', - evalModel: 'alphabetical,not', - outputIndex: 'outputIndex', - projectName: 'Test Project Name', - runName: 'Test Run Name', - }, - signal: undefined, - version: API_VERSIONS.internal.v1, - }); - }); - it('returns error when error is an error', async () => { - const error = 'simulated error'; - (mockHttp.post as jest.Mock).mockImplementation(() => { - throw new Error(error); - }); - - const knowledgeBaseArgs = { - resource: 'a-resource', - http: mockHttp, - }; - - await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); - }); -}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx deleted file mode 100644 index 6581e22e77921a..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; -import { - API_VERSIONS, - GetEvaluateResponse, - PostEvaluateResponse, -} from '@kbn/elastic-assistant-common'; -import { PerformEvaluationParams } from './use_perform_evaluation'; - -export interface PostEvaluationParams { - http: HttpSetup; - evalParams?: PerformEvaluationParams; - signal?: AbortSignal | undefined; -} - -/** - * API call for evaluating models. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.evalParams] - Params necessary for evaluation - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const postEvaluation = async ({ - http, - evalParams, - signal, -}: PostEvaluationParams): Promise => { - try { - const path = `/internal/elastic_assistant/evaluate`; - const query = { - agents: evalParams?.agents.sort()?.join(','), - datasetName: evalParams?.datasetName, - evaluationType: evalParams?.evaluationType.sort()?.join(','), - evalModel: evalParams?.evalModel.sort()?.join(','), - outputIndex: evalParams?.outputIndex, - models: evalParams?.models.sort()?.join(','), - projectName: evalParams?.projectName, - runName: evalParams?.runName, - }; - - return await http.post(path, { - body: JSON.stringify({ - dataset: JSON.parse(evalParams?.dataset ?? '[]'), - evalPrompt: evalParams?.evalPrompt ?? '', - }), - headers: { - 'Content-Type': 'application/json', - }, - query, - signal, - version: API_VERSIONS.internal.v1, - }); - } catch (error) { - return error as IHttpFetchError; - } -}; - -export interface GetEvaluationParams { - http: HttpSetup; - signal?: AbortSignal | undefined; -} - -/** - * API call for fetching evaluation data. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const getEvaluation = async ({ - http, - signal, -}: GetEvaluationParams): Promise => { - try { - const path = `/internal/elastic_assistant/evaluate`; - - return await http.get(path, { - signal, - version: API_VERSIONS.internal.v1, - }); - } catch (error) { - return error as IHttpFetchError; - } -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx deleted file mode 100644 index a37cf18a235ecd..00000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useQuery } from '@tanstack/react-query'; -import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; -import type { IToasts } from '@kbn/core-notifications-browser'; -import { i18n } from '@kbn/i18n'; -import { getEvaluation } from './evaluate'; - -const EVALUATION_DATA_QUERY_KEY = ['elastic-assistant', 'evaluation-data']; - -export interface UseEvaluationDataParams { - http: HttpSetup; - toasts?: IToasts; -} - -/** - * Hook for fetching evaluation data, like available agents, test data, etc - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {IToasts} [options.toasts] - IToasts - * - * @returns {useMutation} mutation hook for setting up the Knowledge Base - */ -export const useEvaluationData = ({ http, toasts }: UseEvaluationDataParams) => { - return useQuery({ - queryKey: EVALUATION_DATA_QUERY_KEY, - queryFn: ({ signal }) => { - // Optional params workaround: see: https://github.com/TanStack/query/issues/1077#issuecomment-1431247266 - return getEvaluation({ http, signal }); - }, - retry: false, - keepPreviousData: true, - // Deprecated, hoist to `queryCache` w/in `QueryClient. See: https://stackoverflow.com/a/76961109 - onError: (error: IHttpFetchError) => { - if (error.name !== 'AbortError') { - toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { - title: i18n.translate('xpack.elasticAssistant.evaluation.fetchEvaluationDataError', { - defaultMessage: 'Error fetching evaluation data...', - }), - }); - } - }, - }); -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx index 09cdf6717ca6c3..f4fe4d7f8a407f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx @@ -27,16 +27,20 @@ import { import { css } from '@emotion/react'; import { FormattedMessage } from '@kbn/i18n-react'; -import type { GetEvaluateResponse, PostEvaluateResponse } from '@kbn/elastic-assistant-common'; import * as i18n from './translations'; import { useAssistantContext } from '../../../assistant_context'; import { useLoadConnectors } from '../../../connectorland/use_load_connectors'; import { getActionTypeTitle, getGenAiConfig } from '../../../connectorland/helpers'; import { PRECONFIGURED_CONNECTOR } from '../../../connectorland/translations'; -import { usePerformEvaluation } from '../../api/evaluate/use_perform_evaluation'; +import { usePerformEvaluation } from './use_perform_evaluation'; import { getApmLink, getDiscoverLink } from './utils'; -import { useEvaluationData } from '../../api/evaluate/use_evaluation_data'; +import { PostEvaluationResponse } from '../../api'; +/** + * See AGENT_EXECUTOR_MAP in `x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts` + * for the agent name -> executor mapping + */ +const DEFAULT_AGENTS = ['DefaultAgentExecutor', 'OpenAIFunctionsExecutor']; const DEFAULT_EVAL_TYPES_OPTIONS = [ { label: 'correctness' }, { label: 'esql-validator', disabled: true }, @@ -61,11 +65,6 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet } = usePerformEvaluation({ http, }); - const { data: evalData } = useEvaluationData({ http }); - const defaultAgents = useMemo( - () => (evalData as GetEvaluateResponse)?.agentExecutors ?? [], - [evalData] - ); // Run Details // Project Name @@ -196,8 +195,8 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet [selectedAgentOptions] ); const agentOptions = useMemo(() => { - return defaultAgents.map((label) => ({ label })); - }, [defaultAgents]); + return DEFAULT_AGENTS.map((label) => ({ label })); + }, []); // Evaluation // Evaluation Type @@ -284,12 +283,12 @@ export const EvaluationSettings: React.FC = React.memo(({ onEvaluationSet ]); const discoverLink = useMemo( - () => getDiscoverLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''), + () => getDiscoverLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''), [basePath, evalResponse] ); const apmLink = useMemo( - () => getApmLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''), + () => getApmLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''), [basePath, evalResponse] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx similarity index 81% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx index f9fdb2e80b7b2d..b0653384805497 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx @@ -7,15 +7,14 @@ import { act, renderHook } from '@testing-library/react-hooks'; import { usePerformEvaluation, UsePerformEvaluationParams } from './use_perform_evaluation'; -import { postEvaluation as _postEvaluation } from './evaluate'; +import { postEvaluation as _postEvaluation } from '../../api'; import { useMutation as _useMutation } from '@tanstack/react-query'; -import { API_VERSIONS } from '@kbn/elastic-assistant-common'; const useMutationMock = _useMutation as jest.Mock; const postEvaluationMock = _postEvaluation as jest.Mock; -jest.mock('./evaluate', () => { - const actual = jest.requireActual('./evaluate'); +jest.mock('../../api', () => { + const actual = jest.requireActual('../../api'); return { ...actual, postEvaluation: jest.fn((...args) => actual.postEvaluation(...args)), @@ -38,7 +37,7 @@ const statusResponse = { }; const http = { - post: jest.fn().mockResolvedValue(statusResponse), + fetch: jest.fn().mockResolvedValue(statusResponse), }; const toasts = { addError: jest.fn(), @@ -54,23 +53,20 @@ describe('usePerformEvaluation', () => { const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps)); await waitForNextUpdate(); - expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + method: 'POST', body: '{"dataset":[],"evalPrompt":""}', headers: { 'Content-Type': 'application/json', }, query: { agents: undefined, - datasetName: undefined, evalModel: undefined, evaluationType: undefined, models: undefined, outputIndex: undefined, - projectName: undefined, - runName: undefined, }, signal: undefined, - version: API_VERSIONS.internal.v1, }); expect(toasts.addError).not.toHaveBeenCalled(); }); @@ -86,8 +82,6 @@ describe('usePerformEvaluation', () => { evaluationType: ['f', 'e'], models: ['h', 'g'], outputIndex: 'outputIndex', - projectName: 'test project', - runName: 'test run', }); return Promise.resolve(res); } catch (e) { @@ -98,23 +92,20 @@ describe('usePerformEvaluation', () => { const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps)); await waitForNextUpdate(); - expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { + method: 'POST', body: '{"dataset":["kewl"],"evalPrompt":"evalPrompt"}', headers: { 'Content-Type': 'application/json', }, query: { agents: 'c,d', - datasetName: undefined, evalModel: 'a,b', evaluationType: 'e,f', models: 'g,h', outputIndex: 'outputIndex', - projectName: 'test project', - runName: 'test run', }, signal: undefined, - version: API_VERSIONS.internal.v1, }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx similarity index 97% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx index 30e95d9d804074..158f7159310ad5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_perform_evaluation.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx @@ -9,7 +9,7 @@ import { useMutation } from '@tanstack/react-query'; import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; import type { IToasts } from '@kbn/core-notifications-browser'; import { i18n } from '@kbn/i18n'; -import { postEvaluation } from './evaluate'; +import { postEvaluation } from '../../api'; const PERFORM_EVALUATION_MUTATION_KEY = ['elastic-assistant', 'perform-evaluation']; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 3551ef6b126c3f..930374567533bf 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -7,9 +7,9 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants'; import { - PostEvaluateRequestBodyInput, - PostEvaluateRequestQueryInput, -} from '@kbn/elastic-assistant-common'; + PostEvaluateBodyInputs, + PostEvaluatePathQueryInputs, +} from '../schemas/evaluate/post_evaluate'; export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -46,8 +46,8 @@ export const getPostEvaluateRequest = ({ body, query, }: { - body: PostEvaluateRequestBodyInput; - query: PostEvaluateRequestQueryInput; + body: PostEvaluateBodyInputs; + query: PostEvaluatePathQueryInputs; }) => requestMock.create({ body, diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts deleted file mode 100644 index b36081ce4bead0..00000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { AgentExecutor } from './types'; -import { callAgentExecutor } from '../execute_custom_llm_chain'; -import { callOpenAIFunctionsExecutor } from './openai_functions_executor'; - -/** - * To support additional Agent Executors from the UI, add them to this map - * and reference your specific AgentExecutor function - */ -export const AGENT_EXECUTOR_MAP: Record = { - DefaultAgentExecutor: callAgentExecutor, - OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor, -}; diff --git a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts index 291aa9d8c25193..54040d3d1b58e6 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts @@ -12,8 +12,8 @@ import { chunk as createChunks } from 'lodash/fp'; import { Logger } from '@kbn/core/server'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer, RunCollectorCallbackHandler } from 'langchain/callbacks'; -import { Dataset } from '@kbn/elastic-assistant-common'; import { AgentExecutorEvaluatorWithMetadata } from '../langchain/executors/types'; +import { Dataset } from '../../schemas/evaluate/post_evaluate'; import { callAgentWithRetry, getMessageFromLangChainResponse } from './utils'; import { ResponseBody } from '../langchain/types'; import { isLangSmithEnabled, writeLangSmithFeedback } from '../../routes/evaluate/utils'; @@ -102,6 +102,7 @@ export const performEvaluation = async ({ const chunk = requestChunks.shift() ?? []; const chunkNumber = totalChunks - requestChunks.length; logger.info(`Prediction request chunk: ${chunkNumber} of ${totalChunks}`); + logger.debug(chunk); // Note, order is kept between chunk and dataset, and is preserved w/ Promise.allSettled const chunkResults = await Promise.allSettled(chunk.map((r) => r.request())); diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index eec0a08ccb8cd3..bbc2c63381fc92 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -43,7 +43,6 @@ import { GetRegisteredTools, } from './services/app_context'; import { getCapabilitiesRoute } from './routes/capabilities/get_capabilities_route'; -import { getEvaluateRoute } from './routes/evaluate/get_evaluate'; interface CreateRouteHandlerContextParams { core: CoreSetup; @@ -125,7 +124,6 @@ export class ElasticAssistantPlugin postActionsConnectorExecuteRoute(router, getElserId); // Evaluate postEvaluateRoute(router, getElserId); - getEvaluateRoute(router); // Capabilities getCapabilitiesRoute(router); return { diff --git a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts index 7c470cdfc2d94b..105e1676fb8089 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts @@ -8,17 +8,12 @@ import { IKibanaResponse, IRouter } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { - API_VERSIONS, - GetCapabilitiesResponse, - INTERNAL_API_ACCESS, -} from '@kbn/elastic-assistant-common'; +import type { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common'; import { CAPABILITIES } from '../../../common/constants'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { buildResponse } from '../../lib/build_response'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -import { buildRouteValidationWithZod } from '../../schemas/common'; /** * Get the assistant capabilities for the requesting plugin @@ -28,7 +23,7 @@ import { buildRouteValidationWithZod } from '../../schemas/common'; export const getCapabilitiesRoute = (router: IRouter) => { router.versioned .get({ - access: INTERNAL_API_ACCESS, + access: 'internal', path: CAPABILITIES, options: { tags: ['access:elasticAssistant'], @@ -36,14 +31,8 @@ export const getCapabilitiesRoute = (router: IRouter> => { const resp = buildResponse(response); diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts deleted file mode 100644 index bc9922ef5f35a6..00000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { type IKibanaResponse, IRouter } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; - -import { - API_VERSIONS, - INTERNAL_API_ACCESS, - GetEvaluateResponse, -} from '@kbn/elastic-assistant-common'; -import { buildResponse } from '../../lib/build_response'; -import { ElasticAssistantRequestHandlerContext } from '../../types'; -import { EVALUATE } from '../../../common/constants'; -import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -import { buildRouteValidationWithZod } from '../../schemas/common'; -import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; - -export const getEvaluateRoute = (router: IRouter) => { - router.versioned - .get({ - access: INTERNAL_API_ACCESS, - path: EVALUATE, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: API_VERSIONS.internal.v1, - validate: { - response: { - 200: { - body: buildRouteValidationWithZod(GetEvaluateResponse), - }, - }, - }, - }, - async (context, request, response): Promise> => { - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - - // Validate evaluation feature is enabled - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); - if (!registeredFeatures.assistantModelEvaluation) { - return response.notFound(); - } - - try { - return response.ok({ body: { agentExecutors: Object.keys(AGENT_EXECUTOR_MAP) } }); - } catch (err) { - logger.error(err); - const error = transformError(err); - - const resp = buildResponse(response); - return resp.error({ - body: { error: error.message }, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts index 64ec69fa5e943d..3ae64f1d89f3b3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts @@ -9,17 +9,17 @@ import { postEvaluateRoute } from './post_evaluate'; import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getPostEvaluateRequest } from '../../__mocks__/request'; -import type { - PostEvaluateRequestBodyInput, - PostEvaluateRequestQueryInput, -} from '@kbn/elastic-assistant-common'; +import { + PostEvaluateBodyInputs, + PostEvaluatePathQueryInputs, +} from '../../schemas/evaluate/post_evaluate'; -const defaultBody: PostEvaluateRequestBodyInput = { +const defaultBody: PostEvaluateBodyInputs = { dataset: undefined, evalPrompt: undefined, }; -const defaultQueryParams: PostEvaluateRequestQueryInput = { +const defaultQueryParams: PostEvaluatePathQueryInputs = { agents: 'agents', datasetName: undefined, evaluationType: undefined, diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index 33d19d6fb61e05..aa041175b75ee0 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -5,23 +5,23 @@ * 2.0. */ -import { type IKibanaResponse, IRouter, KibanaRequest } from '@kbn/core/server'; +import { IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { v4 as uuidv4 } from 'uuid'; -import { - API_VERSIONS, - INTERNAL_API_ACCESS, - PostEvaluateBody, - PostEvaluateRequestQuery, - PostEvaluateResponse, -} from '@kbn/elastic-assistant-common'; import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; +import { buildRouteValidation } from '../../schemas/common'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; import { EVALUATE } from '../../../common/constants'; +import { PostEvaluateBody, PostEvaluatePathQuery } from '../../schemas/evaluate/post_evaluate'; import { performEvaluation } from '../../lib/model_evaluator/evaluation'; -import { AgentExecutorEvaluatorWithMetadata } from '../../lib/langchain/executors/types'; +import { callAgentExecutor } from '../../lib/langchain/execute_custom_llm_chain'; +import { callOpenAIFunctionsExecutor } from '../../lib/langchain/executors/openai_functions_executor'; +import { + AgentExecutor, + AgentExecutorEvaluatorWithMetadata, +} from '../../lib/langchain/executors/types'; import { ActionsClientLlm } from '../../lib/langchain/llm/actions_client_llm'; import { indexEvaluations, @@ -30,8 +30,15 @@ import { import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils'; import { RequestBody } from '../../lib/langchain/types'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -import { buildRouteValidationWithZod } from '../../schemas/common'; -import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; + +/** + * To support additional Agent Executors from the UI, add them to this map + * and reference your specific AgentExecutor function + */ +const AGENT_EXECUTOR_MAP: Record = { + DefaultAgentExecutor: callAgentExecutor, + OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor, +}; const DEFAULT_SIZE = 20; @@ -39,215 +46,200 @@ export const postEvaluateRoute = ( router: IRouter, getElser: GetElser ) => { - router.versioned - .post({ - access: INTERNAL_API_ACCESS, + router.post( + { path: EVALUATE, - options: { - tags: ['access:elasticAssistant'], + validate: { + body: buildRouteValidation(PostEvaluateBody), + query: buildRouteValidation(PostEvaluatePathQuery), }, - }) - .addVersion( - { - version: API_VERSIONS.internal.v1, - validate: { - request: { - body: buildRouteValidationWithZod(PostEvaluateBody), - query: buildRouteValidationWithZod(PostEvaluateRequestQuery), - }, - response: { - 200: { - body: buildRouteValidationWithZod(PostEvaluateResponse), - }, - }, - }, - }, - async (context, request, response): Promise> => { - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; - - // Validate evaluation feature is enabled - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); - if (!registeredFeatures.assistantModelEvaluation) { - return response.notFound(); + }, + async (context, request, response) => { + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; + + // Validate evaluation feature is enabled + const pluginName = getPluginNameFromRequest({ + request, + defaultPluginName: DEFAULT_PLUGIN_NAME, + logger, + }); + const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); + if (!registeredFeatures.assistantModelEvaluation) { + return response.notFound(); + } + + try { + const evaluationId = uuidv4(); + const { + evalModel, + evaluationType, + outputIndex, + datasetName, + projectName = 'default', + runName = evaluationId, + } = request.query; + const { dataset: customDataset = [], evalPrompt } = request.body; + const connectorIds = request.query.models?.split(',') || []; + const agentNames = request.query.agents?.split(',') || []; + + const dataset = + datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; + + logger.info('postEvaluateRoute:'); + logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); + logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); + logger.info(`Evaluation ID: ${evaluationId}`); + + const totalExecutions = connectorIds.length * agentNames.length * dataset.length; + logger.info('Creating agents:'); + logger.info(`\tconnectors/models: ${connectorIds.length}`); + logger.info(`\tagents: ${agentNames.length}`); + logger.info(`\tdataset: ${dataset.length}`); + logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); + if (totalExecutions > 50) { + logger.warn( + `Total baseline agent executions >= 50! This may take a while, and cost some money...` + ); } - try { - const evaluationId = uuidv4(); - const { - evalModel, - evaluationType, - outputIndex, - datasetName, - projectName = 'default', - runName = evaluationId, - } = request.query; - const { dataset: customDataset = [], evalPrompt } = request.body; - const connectorIds = request.query.models?.split(',') || []; - const agentNames = request.query.agents?.split(',') || []; - - const dataset = - datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; - - logger.info('postEvaluateRoute:'); - logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); - logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); - logger.info(`Evaluation ID: ${evaluationId}`); - - const totalExecutions = connectorIds.length * agentNames.length * dataset.length; - logger.info('Creating agents:'); - logger.info(`\tconnectors/models: ${connectorIds.length}`); - logger.info(`\tagents: ${agentNames.length}`); - logger.info(`\tdataset: ${dataset.length}`); - logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); - if (totalExecutions > 50) { - logger.warn( - `Total baseline agent executions >= 50! This may take a while, and cost some money...` - ); - } - - // Get the actions plugin start contract from the request context for the agents - const actions = (await context.elasticAssistant).actions; - - // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm - const actionsClient = await actions.getActionsClientWithRequest(request); - const connectors = await actionsClient.getBulk({ - ids: connectorIds, - throwIfSystemAction: false, - }); + // Get the actions plugin start contract from the request context for the agents + const actions = (await context.elasticAssistant).actions; - // Fetch any tools registered by the request's originating plugin - const assistantTools = (await context.elasticAssistant).getRegisteredTools( - 'securitySolution' - ); + // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm + const actionsClient = await actions.getActionsClientWithRequest(request); + const connectors = await actionsClient.getBulk({ + ids: connectorIds, + throwIfSystemAction: false, + }); - // Get a scoped esClient for passing to the agents for retrieval, and - // writing results to the output index - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - - // Default ELSER model - const elserId = await getElser(request, (await context.core).savedObjects.getClient()); - - // Skeleton request from route to pass to the agents - // params will be passed to the actions executor - const skeletonRequest: KibanaRequest = { - ...request, - body: { - alertsIndexPattern: '', - allow: [], - allowReplacement: [], - params: { - subAction: 'invokeAI', - subActionParams: { - messages: [], - }, + // Fetch any tools registered by the request's originating plugin + const assistantTools = (await context.elasticAssistant).getRegisteredTools( + 'securitySolution' + ); + + // Get a scoped esClient for passing to the agents for retrieval, and + // writing results to the output index + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + // Default ELSER model + const elserId = await getElser(request, (await context.core).savedObjects.getClient()); + + // Skeleton request from route to pass to the agents + // params will be passed to the actions executor + const skeletonRequest: KibanaRequest = { + ...request, + body: { + alertsIndexPattern: '', + allow: [], + allowReplacement: [], + params: { + subAction: 'invokeAI', + subActionParams: { + messages: [], }, - replacements: {}, - size: DEFAULT_SIZE, - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: true, }, - }; - - // Create an array of executor functions to call in batches - // One for each connector/model + agent combination - // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator - const agents: AgentExecutorEvaluatorWithMetadata[] = []; - connectorIds.forEach((connectorId) => { - agentNames.forEach((agentName) => { - logger.info(`Creating agent: ${connectorId} + ${agentName}`); - const llmType = getLlmType(connectorId, connectors); - const connectorName = - getConnectorName(connectorId, connectors) ?? '[unknown connector]'; - const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; - agents.push({ - agentEvaluator: (langChainMessages, exampleId) => - AGENT_EXECUTOR_MAP[agentName]({ - actions, - isEnabledKnowledgeBase: true, - assistantTools, - connectorId, - esClient, - elserId, - langChainMessages, - llmType, - logger, - request: skeletonRequest, - kbResource: ESQL_RESOURCE, - telemetry, - traceOptions: { - exampleId, - projectName, - runName: detailedRunName, - evaluationId, - tags: [ - 'security-assistant-prediction', - ...(connectorName != null ? [connectorName] : []), - runName, - ], - tracers: getLangSmithTracer(detailedRunName, exampleId, logger), - }, - }), - metadata: { - connectorName, - runName: detailedRunName, - }, - }); - }); - }); - logger.info(`Agents created: ${agents.length}`); - - // Evaluator Model is optional to support just running predictions - const evaluatorModel = - evalModel == null || evalModel === '' - ? undefined - : new ActionsClientLlm({ + replacements: {}, + size: DEFAULT_SIZE, + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: true, + }, + }; + + // Create an array of executor functions to call in batches + // One for each connector/model + agent combination + // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator + const agents: AgentExecutorEvaluatorWithMetadata[] = []; + connectorIds.forEach((connectorId) => { + agentNames.forEach((agentName) => { + logger.info(`Creating agent: ${connectorId} + ${agentName}`); + const llmType = getLlmType(connectorId, connectors); + const connectorName = + getConnectorName(connectorId, connectors) ?? '[unknown connector]'; + const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; + agents.push({ + agentEvaluator: (langChainMessages, exampleId) => + AGENT_EXECUTOR_MAP[agentName]({ actions, - connectorId: evalModel, - request: skeletonRequest, + isEnabledKnowledgeBase: true, + assistantTools, + connectorId, + esClient, + elserId, + langChainMessages, + llmType, logger, - }); - - const { evaluationResults, evaluationSummary } = await performEvaluation({ - agentExecutorEvaluators: agents, - dataset, - evaluationId, - evaluatorModel, - evaluationPrompt: evalPrompt, - evaluationType, - logger, - runName, + request: skeletonRequest, + kbResource: ESQL_RESOURCE, + telemetry, + traceOptions: { + exampleId, + projectName, + runName: detailedRunName, + evaluationId, + tags: [ + 'security-assistant-prediction', + ...(connectorName != null ? [connectorName] : []), + runName, + ], + tracers: getLangSmithTracer(detailedRunName, exampleId, logger), + }, + }), + metadata: { + connectorName, + runName: detailedRunName, + }, + }); }); + }); + logger.info(`Agents created: ${agents.length}`); + + // Evaluator Model is optional to support just running predictions + const evaluatorModel = + evalModel == null || evalModel === '' + ? undefined + : new ActionsClientLlm({ + actions, + connectorId: evalModel, + request: skeletonRequest, + logger, + }); - logger.info(`Writing evaluation results to index: ${outputIndex}`); - await setupEvaluationIndex({ esClient, index: outputIndex, logger }); - await indexEvaluations({ - esClient, - evaluationResults, - evaluationSummary, - index: outputIndex, - logger, - }); + const { evaluationResults, evaluationSummary } = await performEvaluation({ + agentExecutorEvaluators: agents, + dataset, + evaluationId, + evaluatorModel, + evaluationPrompt: evalPrompt, + evaluationType, + logger, + runName, + }); - return response.ok({ - body: { evaluationId, success: true }, - }); - } catch (err) { - logger.error(err); - const error = transformError(err); - - const resp = buildResponse(response); - return resp.error({ - body: { success: false, error: error.message }, - statusCode: error.statusCode, - }); - } + logger.info(`Writing evaluation results to index: ${outputIndex}`); + await setupEvaluationIndex({ esClient, index: outputIndex, logger }); + await indexEvaluations({ + esClient, + evaluationResults, + evaluationSummary, + index: outputIndex, + logger, + }); + + return response.ok({ + body: { evaluationId, success: true }, + }); + } catch (err) { + logger.error(err); + const error = transformError(err); + + const resp = buildResponse(response); + return resp.error({ + body: { success: false, error: error.message }, + statusCode: error.statusCode, + }); } - ); + } + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts index 11f8cb9c2f692b..550e89667256e3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts @@ -12,7 +12,7 @@ import type { Logger } from '@kbn/core/server'; import type { Run } from 'langsmith/schemas'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer } from 'langchain/callbacks'; -import { Dataset } from '@kbn/elastic-assistant-common'; +import { Dataset } from '../../schemas/evaluate/post_evaluate'; /** * Returns the LangChain `llmType` for the given connectorId/connectors diff --git a/x-pack/plugins/elastic_assistant/server/schemas/common.ts b/x-pack/plugins/elastic_assistant/server/schemas/common.ts index 5e847aef69fc08..00e97a9326c5e6 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/common.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/common.ts @@ -14,8 +14,6 @@ import type { RouteValidationResultFactory, RouteValidationError, } from '@kbn/core/server'; -import type { TypeOf, ZodType } from 'zod'; -import { stringifyZodError } from '@kbn/zod-helpers'; type RequestValidationResult = | { @@ -38,14 +36,3 @@ export const buildRouteValidation = (validatedInput: A) => validationResult.ok(validatedInput) ) ); - -export const buildRouteValidationWithZod = - >(schema: T): RouteValidationFunction => - (inputValue: unknown, validationResult: RouteValidationResultFactory) => { - const decoded = schema.safeParse(inputValue); - if (decoded.success) { - return validationResult.ok(decoded.data); - } else { - return validationResult.badRequest(stringifyZodError(decoded.error)); - } - }; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts new file mode 100644 index 00000000000000..f520bf9bf93b6b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +/** Validates Output Index starts with `.kibana-elastic-ai-assistant-` */ +const outputIndex = new t.Type( + 'OutputIndexPrefixed', + (input): input is string => + typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-'), + (input, context) => + typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-') + ? t.success(input) + : t.failure( + input, + context, + `Type error: Output Index does not start with '.kibana-elastic-ai-assistant-'` + ), + t.identity +); + +/** Validates the URL path of a POST request to the `/evaluate` endpoint */ +export const PostEvaluatePathQuery = t.type({ + agents: t.string, + datasetName: t.union([t.string, t.undefined]), + evaluationType: t.union([t.string, t.undefined]), + evalModel: t.union([t.string, t.undefined]), + models: t.string, + outputIndex, + projectName: t.union([t.string, t.undefined]), + runName: t.union([t.string, t.undefined]), +}); + +export type PostEvaluatePathQueryInputs = t.TypeOf; + +export type DatasetItem = t.TypeOf; +export const DatasetItem = t.type({ + id: t.union([t.string, t.undefined]), + input: t.string, + reference: t.string, + tags: t.union([t.array(t.string), t.undefined]), + prediction: t.union([t.string, t.undefined]), +}); + +export type Dataset = t.TypeOf; +export const Dataset = t.array(DatasetItem); + +/** Validates the body of a POST request to the `/evaluate` endpoint */ +export const PostEvaluateBody = t.type({ + dataset: t.union([Dataset, t.undefined]), + evalPrompt: t.union([t.string, t.undefined]), +}); + +export type PostEvaluateBodyInputs = t.TypeOf; diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 2717da8d33a3ac..dfca7893b20365 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -35,7 +35,6 @@ "@kbn/core-analytics-server", "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", - "@kbn/zod-helpers", ], "exclude": [ "target/**/*", From b2bde8b8de0814a3b83a4a25cd43f9099ca681d0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 31 Jan 2024 13:33:38 -0600 Subject: [PATCH 17/20] Revert "skip failing test suite (#175984)" This reverts commit 399a3c42aa4d23bc4aa5699d6b221b5b98cc2cf4. --- .../common/components/charts/barchart.test.tsx | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/charts/barchart.test.tsx b/x-pack/plugins/security_solution/public/common/components/charts/barchart.test.tsx index bea9752d6e7703..38d3aeb37bac95 100644 --- a/x-pack/plugins/security_solution/public/common/components/charts/barchart.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/charts/barchart.test.tsx @@ -139,8 +139,7 @@ const mockConfig = { customHeight: 324, }; -// Failing: See https://github.com/elastic/kibana/issues/175984 -describe.skip('BarChartBaseComponent', () => { +describe('BarChartBaseComponent', () => { let shallowWrapper: ShallowWrapper; const mockBarChartData: ChartSeriesData[] = [ { @@ -297,8 +296,7 @@ describe.skip('BarChartBaseComponent', () => { }); }); -// Failing: See https://github.com/elastic/kibana/issues/175984 -describe.skip.each(chartDataSets)('BarChart with valid data [%o]', (data) => { +describe.each(chartDataSets)('BarChart with valid data [%o]', (data) => { let shallowWrapper: ShallowWrapper; beforeAll(() => { @@ -315,8 +313,7 @@ describe.skip.each(chartDataSets)('BarChart with valid data [%o]', (data) => { }); }); -// Failing: See https://github.com/elastic/kibana/issues/175984 -describe.skip.each(chartDataSets)('BarChart with stackByField', () => { +describe.each(chartDataSets)('BarChart with stackByField', () => { let wrapper: ReactWrapper; const data = [ @@ -391,8 +388,7 @@ describe.skip.each(chartDataSets)('BarChart with stackByField', () => { }); }); -// Failing: See https://github.com/elastic/kibana/issues/175984 -describe.skip.each(chartDataSets)('BarChart with custom color', () => { +describe.each(chartDataSets)('BarChart with custom color', () => { let wrapper: ReactWrapper; const data = [ @@ -455,8 +451,7 @@ describe.skip.each(chartDataSets)('BarChart with custom color', () => { }); }); -// Failing: See https://github.com/elastic/kibana/issues/175984 -describe.skip.each(chartHolderDataSets)('BarChart with invalid data [%o]', (data) => { +describe.each(chartHolderDataSets)('BarChart with invalid data [%o]', (data) => { let shallowWrapper: ShallowWrapper; beforeAll(() => { From 4b56efeeae4d9579fe255706e58364f4f994124e Mon Sep 17 00:00:00 2001 From: Philippe Oberti Date: Wed, 31 Jan 2024 15:33:23 -0600 Subject: [PATCH 18/20] [Security Solution][Timeline] refactor flyout folder (#175747) --- .../alerts_response_console.cy.ts | 2 +- .../flyout/action_menu/index.test.tsx | 114 ----------- .../components/flyout/action_menu/index.tsx | 85 -------- .../components/flyout/header/index.tsx | 169 ---------------- .../components/flyout/header/selectors.ts | 21 -- .../timelines/components/flyout/pane/index.ts | 8 - .../components/flyout/pane/pane.test.tsx | 62 ------ .../timelines/components/flyout/pane/pane.tsx | 62 ------ .../components/flyout/pane/translations.ts | 15 -- .../{flyout/pane => modal}/custom_portal.tsx | 51 ++--- .../components/modal/header/index.test.tsx | 120 +++++++++++ .../components/modal/header/index.tsx | 191 ++++++++++++++++++ .../index.styles.tsx} | 17 +- .../timelines/components/modal/index.test.tsx | 61 ++++++ .../timelines/components/modal/index.tsx | 75 +++++++ .../{flyout/header => modal}/translations.ts | 0 .../timelines/components/timeline/index.tsx | 14 +- .../public/timelines/store/selectors.ts | 55 ++++- .../public/timelines/wrapper/index.test.tsx | 3 +- .../public/timelines/wrapper/index.tsx | 4 +- .../cypress/screens/timeline.ts | 2 +- .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../cypress/screens/security_main.ts | 2 - .../cypress/screens/timeline.ts | 14 +- .../cypress/tasks/security_main.ts | 7 +- .../page_objects/timeline/index.ts | 2 +- 28 files changed, 544 insertions(+), 615 deletions(-) delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.test.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/header/selectors.ts delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.ts delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.test.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/flyout/pane/translations.ts rename x-pack/plugins/security_solution/public/timelines/components/{flyout/pane => modal}/custom_portal.tsx (54%) create mode 100644 x-pack/plugins/security_solution/public/timelines/components/modal/header/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/modal/header/index.tsx rename x-pack/plugins/security_solution/public/timelines/components/{flyout/pane/pane.styles.tsx => modal/index.styles.tsx} (84%) create mode 100644 x-pack/plugins/security_solution/public/timelines/components/modal/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/modal/index.tsx rename x-pack/plugins/security_solution/public/timelines/components/{flyout/header => modal}/translations.ts (100%) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts index 0a91f0e2af5528..84fd3f1bbea645 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts @@ -104,7 +104,7 @@ describe( getAlertsTableRows().should('have.length.greaterThan', 0); openInvestigateInTimelineView(); - cy.getByTestSubj('timeline-flyout').within(() => { + cy.getByTestSubj('timeline-container').within(() => { openAlertDetailsView(); }); openResponderFromEndpointAlertDetails(); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.test.tsx deleted file mode 100644 index 54eb8cf2769a94..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.test.tsx +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { render, screen } from '@testing-library/react'; - -import { TestProviders, mockIndexNames, mockIndexPattern } from '../../../../common/mock'; -import { useSourcererDataView } from '../../../../common/containers/sourcerer'; -import { allCasesPermissions, readCasesPermissions } from '../../../../cases_test_utils'; -import { mockBrowserFields } from '../../../../common/containers/source/mock'; -import { TimelineActionMenu } from '.'; -import { TimelineId, TimelineTabs } from '../../../../../common/types'; -import { useKibana as mockUseKibana } from '../../../../common/lib/kibana/__mocks__'; - -const mockUseSourcererDataView: jest.Mock = useSourcererDataView as jest.Mock; -const mockedUseKibana = mockUseKibana(); -const mockCanUseCases = jest.fn(); - -jest.mock('../../../../common/containers/sourcerer'); - -jest.mock('../../../../common/lib/kibana/kibana_react', () => { - const original = jest.requireActual('../../../../common/lib/kibana/kibana_react'); - - return { - ...original, - useKibana: () => ({ - ...mockedUseKibana, - services: { - ...mockedUseKibana.services, - cases: { - ...mockedUseKibana.services.cases, - helpers: { canUseCases: mockCanUseCases }, - }, - }, - application: { - capabilities: { - navLinks: {}, - management: {}, - catalogue: {}, - actions: { show: true, crud: true }, - }, - }, - }), - }; -}); - -jest.mock('@kbn/i18n-react', () => { - const originalModule = jest.requireActual('@kbn/i18n-react'); - const FormattedRelative = jest.fn().mockImplementation(() => '20 hours ago'); - - return { - ...originalModule, - FormattedRelative, - }; -}); - -const sourcererDefaultValue = { - sourcererDefaultValue: mockBrowserFields, - indexPattern: mockIndexPattern, - loading: false, - selectedPatterns: mockIndexNames, -}; - -describe('Action menu', () => { - beforeEach(() => { - // Mocking these services is required for the header component to render. - mockUseSourcererDataView.mockImplementation(() => sourcererDefaultValue); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('AddToCaseButton', () => { - it('renders the button when the user has create and read permissions', () => { - mockCanUseCases.mockReturnValue(allCasesPermissions()); - - render( - - - - ); - - expect( - screen.getByTestId('timeline-modal-attach-to-case-dropdown-button') - ).toBeInTheDocument(); - }); - - it('does not render the button when the user does not have create permissions', () => { - mockCanUseCases.mockReturnValue(readCasesPermissions()); - - render( - - - - ); - - expect( - screen.queryByTestId('timeline-modal-attach-to-case-dropdown-button') - ).not.toBeInTheDocument(); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.tsx deleted file mode 100644 index 1fc6e41ec43f3b..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/action_menu/index.tsx +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import React from 'react'; -import styled from 'styled-components'; -import { AttachToCaseButton } from '../../modal/actions/attach_to_case_button'; -import { useKibana } from '../../../../common/lib/kibana/kibana_react'; -import { APP_ID } from '../../../../../common'; -import type { TimelineTabs } from '../../../../../common/types'; -import { InspectButton } from '../../../../common/components/inspect'; -import { InputsModelId } from '../../../../common/store/inputs/constants'; -import { NewTimelineButton } from '../../modal/actions/new_timeline_button'; -import { SaveTimelineButton } from '../../modal/actions/save_timeline_button'; -import { OpenTimelineButton } from '../../modal/actions/open_timeline_button'; -import { TIMELINE_TOUR_CONFIG_ANCHORS } from '../../timeline/tour/step_config'; - -interface TimelineActionMenuProps { - mode?: 'compact' | 'normal'; - timelineId: string; - isInspectButtonDisabled: boolean; - activeTab: TimelineTabs; -} - -const VerticalDivider = styled.span` - width: 0px; - height: 20px; - border-left: 1px solid ${({ theme }) => theme.eui.euiColorLightShade}; -`; - -const TimelineActionMenuComponent = ({ - mode = 'normal', - timelineId, - activeTab, - isInspectButtonDisabled, -}: TimelineActionMenuProps) => { - const { cases } = useKibana().services; - const userCasesPermissions = cases.helpers.canUseCases([APP_ID]); - - return ( - - - - - - - - - - - {userCasesPermissions.create && userCasesPermissions.read ? ( - <> - - - - - - - - ) : null} - - - - - ); -}; - -export const TimelineActionMenu = React.memo(TimelineActionMenuComponent); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx deleted file mode 100644 index 930acce9f8bd30..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - EuiFlexGroup, - EuiFlexItem, - EuiPanel, - EuiToolTip, - EuiButtonIcon, - EuiText, -} from '@elastic/eui'; -import React, { useCallback, useMemo } from 'react'; -import { isEmpty, get, pick } from 'lodash/fp'; -import { useDispatch, useSelector } from 'react-redux'; -import { getEsQueryConfig } from '@kbn/data-plugin/common'; -import { euiStyled } from '@kbn/kibana-react-plugin/common'; -import { selectTitleByTimelineById } from '../../../store/selectors'; -import { createHistoryEntry } from '../../../../common/utils/global_query_string/helpers'; -import { useDeepEqualSelector } from '../../../../common/hooks/use_selector'; -import { timelineActions, timelineSelectors } from '../../../store'; -import type { State } from '../../../../common/store'; -import { useKibana } from '../../../../common/lib/kibana'; -import { useSourcererDataView } from '../../../../common/containers/sourcerer'; -import { combineQueries } from '../../../../common/lib/kuery'; -import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; -import * as i18n from './translations'; -import { TimelineActionMenu } from '../action_menu'; -import { AddToFavoritesButton } from '../../add_to_favorites'; -import { TimelineSaveStatus } from '../../save_status'; -import { timelineDefaults } from '../../../store/defaults'; - -interface FlyoutHeaderPanelProps { - timelineId: string; -} - -const whiteSpaceNoWrapCSS = { 'white-space': 'nowrap' }; -const autoOverflowXCSS = { 'overflow-x': 'auto' }; - -const TimelinePanel = euiStyled(EuiPanel)<{ $isOpen?: boolean }>` - backgroundColor: ${(props) => props.theme.eui.euiColorEmptyShade}; - color: ${(props) => props.theme.eui.euiTextColor}; - padding-inline: ${(props) => props.theme.eui.euiSizeM}; - border-radius: ${({ $isOpen, theme }) => ($isOpen ? theme.eui.euiBorderRadius : '0px')}; -`; - -const FlyoutHeaderPanelComponent: React.FC = ({ timelineId }) => { - const dispatch = useDispatch(); - const { browserFields, indexPattern } = useSourcererDataView(SourcererScopeName.timeline); - const { uiSettings } = useKibana().services; - const esQueryConfig = useMemo(() => getEsQueryConfig(uiSettings), [uiSettings]); - - const title = useSelector((state: State) => selectTitleByTimelineById(state, timelineId)); - - const getTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); - const { activeTab, dataProviders, kqlQuery, timelineType, show, filters, kqlMode } = - useDeepEqualSelector((state) => - pick( - ['activeTab', 'dataProviders', 'kqlQuery', 'timelineType', 'show', 'filters', 'kqlMode'], - getTimeline(state, timelineId) ?? timelineDefaults - ) - ); - const isDataInTimeline = useMemo( - () => !isEmpty(dataProviders) || !isEmpty(get('filterQuery.kuery.expression', kqlQuery)), - [dataProviders, kqlQuery] - ); - - const getKqlQueryTimeline = useMemo(() => timelineSelectors.getKqlFilterQuerySelector(), []); - - const kqlQueryTimeline = useSelector((state: State) => getKqlQueryTimeline(state, timelineId)); - - const kqlQueryExpression = - isEmpty(dataProviders) && isEmpty(kqlQueryTimeline) && timelineType === 'template' - ? ' ' - : kqlQueryTimeline ?? ''; - - const kqlQueryObj = useMemo( - () => ({ query: kqlQueryExpression, language: 'kuery' }), - [kqlQueryExpression] - ); - - const combinedQueries = useMemo( - () => - combineQueries({ - config: esQueryConfig, - dataProviders, - indexPattern, - browserFields, - filters: filters ? filters : [], - kqlQuery: kqlQueryObj, - kqlMode, - }), - [browserFields, dataProviders, esQueryConfig, filters, indexPattern, kqlMode, kqlQueryObj] - ); - - const handleClose = useCallback(() => { - createHistoryEntry(); - dispatch(timelineActions.showTimeline({ id: timelineId, show: false })); - }, [dispatch, timelineId]); - - return ( - - - - - - - - - -

{title}

- - - - - - - - {show && ( - - - - - - - - - - - )} - - - ); -}; - -export const FlyoutHeaderPanel = React.memo(FlyoutHeaderPanelComponent); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/selectors.ts b/x-pack/plugins/security_solution/public/timelines/components/flyout/header/selectors.ts deleted file mode 100644 index 441ae5fed1a7e6..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/selectors.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { createSelector } from 'reselect'; - -import { TimelineStatus } from '../../../../../common/api/timeline'; -import { timelineSelectors } from '../../../store'; - -export const getTimelineStatusByIdSelector = () => - createSelector(timelineSelectors.selectTimeline, (timeline) => ({ - status: timeline?.status ?? TimelineStatus.draft, - updated: timeline?.updated ?? undefined, - isSaving: timeline?.isSaving ?? undefined, - isLoading: timeline?.isLoading ?? undefined, - changed: timeline?.changed ?? undefined, - show: timeline?.show ?? undefined, - })); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.ts b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.ts deleted file mode 100644 index 7cfdf2a31aca27..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { Pane } from './pane'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.test.tsx deleted file mode 100644 index 08d1465b08fbaf..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.test.tsx +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { render } from '@testing-library/react'; -import React from 'react'; - -import { TestProviders } from '../../../../common/mock'; -import { TimelineId } from '../../../../../common/types/timeline'; -import { Pane } from '.'; - -jest.mock('../../timeline', () => ({ - StatefulTimeline: () =>
, -})); - -const mockIsFullScreen = jest.fn(() => false); -jest.mock('../../../../common/store/selectors', () => ({ - inputsSelectors: { timelineFullScreenSelector: () => mockIsFullScreen() }, -})); - -describe('Pane', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should render the timeline', async () => { - const wrapper = render( - - - - ); - - expect(wrapper.getByTestId('StatefulTimelineMock')).toBeInTheDocument(); - }); - - it('should render without fullscreen className', async () => { - mockIsFullScreen.mockReturnValue(false); - const wrapper = render( - - - - ); - - expect(wrapper.getByTestId('timeline-wrapper')).not.toHaveClass( - 'timeline-wrapper--full-screen' - ); - }); - - it('should render with fullscreen className', async () => { - mockIsFullScreen.mockReturnValue(true); - const wrapper = render( - - - - ); - - expect(wrapper.getByTestId('timeline-wrapper')).toHaveClass('timeline-wrapper--full-screen'); - }); -}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.tsx deleted file mode 100644 index 853c88044cd8b9..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.tsx +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { useRef } from 'react'; -import classNames from 'classnames'; -import { StatefulTimeline } from '../../timeline'; -import type { TimelineId } from '../../../../../common/types/timeline'; -import * as i18n from './translations'; -import { defaultRowRenderers } from '../../timeline/body/renderers'; -import { DefaultCellRenderer } from '../../timeline/cell_rendering/default_cell_renderer'; -import { EuiPortal } from './custom_portal'; -import { useShallowEqualSelector } from '../../../../common/hooks/use_selector'; -import { inputsSelectors } from '../../../../common/store/selectors'; -import { usePaneStyles, OverflowHiddenGlobalStyles } from './pane.styles'; - -interface FlyoutPaneComponentProps { - timelineId: TimelineId; - visible?: boolean; -} - -const FlyoutPaneComponent: React.FC = ({ - timelineId, - visible = true, -}) => { - const ref = useRef(null); - const isFullScreen = useShallowEqualSelector(inputsSelectors.timelineFullScreenSelector) ?? false; - - const styles = usePaneStyles(); - const wrapperClassName = classNames('timeline-wrapper', styles, { - 'timeline-wrapper--full-screen': isFullScreen, - 'timeline-wrapper--hidden': !visible, - }); - - return ( -
- -
-
- -
-
- - {visible && } -
- ); -}; - -export const Pane = React.memo(FlyoutPaneComponent); - -Pane.displayName = 'Pane'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/translations.ts b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/translations.ts deleted file mode 100644 index 6a0d15af819fac..00000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/translations.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; - -export const TIMELINE_DESCRIPTION = i18n.translate( - 'xpack.securitySolution.timeline.flyout.pane.timelinePropertiesAriaLabel', - { - defaultMessage: 'Timeline Properties', - } -); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/custom_portal.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/custom_portal.tsx similarity index 54% rename from x-pack/plugins/security_solution/public/timelines/components/flyout/pane/custom_portal.tsx rename to x-pack/plugins/security_solution/public/timelines/components/modal/custom_portal.tsx index 861f90851c32c4..87c43436207909 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/custom_portal.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/custom_portal.tsx @@ -15,81 +15,62 @@ import type { ReactNode } from 'react'; import { Component } from 'react'; import { createPortal } from 'react-dom'; -interface InsertPositionsMap { - after: InsertPosition; - before: InsertPosition; -} - -export const insertPositions: InsertPositionsMap = { - after: 'afterend', - before: 'beforebegin', -}; - -export interface EuiPortalProps { +export interface CustomEuiPortalProps { /** * ReactNode to render as this component's content */ children: ReactNode; - insert?: { sibling: HTMLElement | null; position: 'before' | 'after' }; - portalRef?: (ref: HTMLDivElement | null) => void; + /** + * Sibling is the React node or HTMLElement to insert the portal next to + * Position specifies the portal's relative position, either before or after + */ + sibling: HTMLDivElement | null; } -export class EuiPortal extends Component { +export class CustomEuiPortal extends Component { portalNode: HTMLDivElement | null = null; - constructor(props: EuiPortalProps) { + constructor(props: CustomEuiPortalProps) { super(props); if (typeof window === 'undefined') return; // Prevent SSR errors - const { insert } = this.props; + const { sibling } = this.props; this.portalNode = document.createElement('div'); this.portalNode.dataset.euiportal = 'true'; - if (insert == null || insert.sibling == null) { + if (sibling == null) { // no insertion defined, append to body document.body.appendChild(this.portalNode); } else { // inserting before or after an element - const { sibling, position } = insert; - sibling.insertAdjacentElement(insertPositions[position], this.portalNode); + sibling.insertAdjacentElement('afterend', this.portalNode); } } - componentDidMount() { - this.updatePortalRef(this.portalNode); - } - componentWillUnmount() { if (this.portalNode?.parentNode) { this.portalNode.parentNode.removeChild(this.portalNode); } - this.updatePortalRef(null); } - componentDidUpdate(prevProps: Readonly): void { - if (!deepEqual(prevProps.insert, this.props.insert) && this.portalNode?.parentNode) { + componentDidUpdate(prevProps: Readonly): void { + if (!deepEqual(prevProps.sibling, this.props.sibling) && this.portalNode?.parentNode) { this.portalNode.parentNode.removeChild(this.portalNode); } if (this.portalNode) { - if (this.props.insert == null || this.props.insert.sibling == null) { + if (this.props == null || this.props.sibling == null) { // no insertion defined, append to body document.body.appendChild(this.portalNode); } else { // inserting before or after an element - const { sibling, position } = this.props.insert; - sibling.insertAdjacentElement(insertPositions[position], this.portalNode); + const { sibling } = this.props; + sibling.insertAdjacentElement('afterend', this.portalNode); } } } - updatePortalRef(ref: HTMLDivElement | null) { - if (this.props.portalRef) { - this.props.portalRef(ref); - } - } - render() { return this.portalNode ? createPortal(this.props.children, this.portalNode) : null; } diff --git a/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.test.tsx new file mode 100644 index 00000000000000..15246435852fb2 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.test.tsx @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { TestProviders } from '../../../../common/mock'; +import { TimelineModalHeader } from '.'; +import { render } from '@testing-library/react'; +import { useSourcererDataView } from '../../../../common/containers/sourcerer'; +import { useCreateTimeline } from '../../../hooks/use_create_timeline'; +import { useInspect } from '../../../../common/components/inspect/use_inspect'; +import { useKibana } from '../../../../common/lib/kibana'; +import { timelineActions } from '../../../store'; + +jest.mock('../../../../common/containers/sourcerer'); +jest.mock('../../../hooks/use_create_timeline'); +jest.mock('../../../../common/components/inspect/use_inspect'); +jest.mock('../../../../common/lib/kibana'); + +const mockGetState = jest.fn(); +jest.mock('react-redux', () => { + const actual = jest.requireActual('react-redux'); + return { + ...actual, + useDispatch: jest.fn(), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + useSelector: (selector: any) => + selector({ + timeline: { + timelineById: { + 'timeline-1': { + ...mockGetState(), + }, + }, + }, + }), + }; +}); + +const timelineId = 'timeline-1'; +const renderTimelineModalHeader = () => + render( + + + + ); + +describe('TimelineModalHeader', () => { + (useCreateTimeline as jest.Mock).mockReturnValue(jest.fn()); + (useInspect as jest.Mock).mockReturnValue(jest.fn()); + + it('should render all dom elements', () => { + (useSourcererDataView as jest.Mock).mockReturnValue({ + browserFields: {}, + indexPattern: { fields: [], title: '' }, + }); + + const { getByTestId, getByText } = renderTimelineModalHeader(); + + expect(getByTestId('timeline-favorite-empty-star')).toBeInTheDocument(); + expect(getByText('Untitled timeline')).toBeInTheDocument(); + expect(getByTestId('timeline-save-status')).toBeInTheDocument(); + expect(getByTestId('timeline-modal-header-actions')).toBeInTheDocument(); + expect(getByTestId('timeline-modal-new-timeline-dropdown-button')).toBeInTheDocument(); + expect(getByTestId('timeline-modal-open-timeline-button')).toBeInTheDocument(); + expect(getByTestId('inspect-empty-button')).toBeInTheDocument(); + expect(getByTestId('timeline-modal-save-timeline')).toBeInTheDocument(); + expect(getByTestId('timeline-modal-header-close-button')).toBeInTheDocument(); + }); + + it('should show attach to case if user has the correct permissions', () => { + (useSourcererDataView as jest.Mock).mockReturnValue({ + browserFields: {}, + indexPattern: { fields: [], title: '' }, + }); + (useKibana as jest.Mock).mockReturnValue({ + services: { + application: { + navigateToApp: jest.fn(), + }, + cases: { + helpers: { + canUseCases: jest.fn().mockReturnValue({ + create: true, + read: true, + }), + }, + }, + uiSettings: { + get: jest.fn(), + }, + }, + }); + + const { getByTestId } = renderTimelineModalHeader(); + + expect(getByTestId('timeline-modal-attach-to-case-dropdown-button')).toBeInTheDocument(); + }); + + it('should call showTimeline action when closing timeline', () => { + (useSourcererDataView as jest.Mock).mockReturnValue({ + browserFields: {}, + indexPattern: { fields: [], title: '' }, + }); + + const spy = jest.spyOn(timelineActions, 'showTimeline'); + + const { getByTestId } = renderTimelineModalHeader(); + + getByTestId('timeline-modal-header-close-button').click(); + + expect(spy).toHaveBeenCalledWith({ + id: timelineId, + show: false, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.tsx new file mode 100644 index 00000000000000..1fd1dc77c80a79 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/header/index.tsx @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiToolTip, + EuiButtonIcon, + EuiText, +} from '@elastic/eui'; +import React, { useCallback, useMemo } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { getEsQueryConfig } from '@kbn/data-plugin/common'; +import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import styled from 'styled-components'; +import { TIMELINE_TOUR_CONFIG_ANCHORS } from '../../timeline/tour/step_config'; +import { NewTimelineButton } from '../actions/new_timeline_button'; +import { OpenTimelineButton } from '../actions/open_timeline_button'; +import { APP_ID } from '../../../../../common'; +import { + selectDataInTimeline, + selectKqlQuery, + selectTimelineById, + selectTitleByTimelineById, +} from '../../../store/selectors'; +import { createHistoryEntry } from '../../../../common/utils/global_query_string/helpers'; +import { timelineActions } from '../../../store'; +import type { State } from '../../../../common/store'; +import { useKibana } from '../../../../common/lib/kibana'; +import { useSourcererDataView } from '../../../../common/containers/sourcerer'; +import { combineQueries } from '../../../../common/lib/kuery'; +import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; +import * as i18n from '../translations'; +import { AddToFavoritesButton } from '../../add_to_favorites'; +import { TimelineSaveStatus } from '../../save_status'; +import { InspectButton } from '../../../../common/components/inspect'; +import { InputsModelId } from '../../../../common/store/inputs/constants'; +import { AttachToCaseButton } from '../actions/attach_to_case_button'; +import { SaveTimelineButton } from '../actions/save_timeline_button'; + +const whiteSpaceNoWrapCSS = { 'white-space': 'nowrap' }; +const autoOverflowXCSS = { 'overflow-x': 'auto' }; +const VerticalDivider = styled.span` + width: 0; + height: 20px; + border-left: 1px solid ${({ theme }) => theme.eui.euiColorLightShade}; +`; +const TimelinePanel = euiStyled(EuiPanel)` + backgroundColor: ${(props) => props.theme.eui.euiColorEmptyShade}; + color: ${(props) => props.theme.eui.euiTextColor}; + padding-inline: ${(props) => props.theme.eui.euiSizeM}; + border-radius: ${({ theme }) => theme.eui.euiBorderRadius}; +`; + +interface FlyoutHeaderPanelProps { + /** + * Id of the timeline to be displayed within the modal + */ + timelineId: string; +} + +/** + * Component rendered at the top of the timeline modal. It contains the timeline title, all the action buttons (save, open, favorite...) and the close button + */ +export const TimelineModalHeader = React.memo(({ timelineId }) => { + const dispatch = useDispatch(); + const { browserFields, indexPattern } = useSourcererDataView(SourcererScopeName.timeline); + const { cases, uiSettings } = useKibana().services; + const esQueryConfig = useMemo(() => getEsQueryConfig(uiSettings), [uiSettings]); + const userCasesPermissions = cases.helpers.canUseCases([APP_ID]); + + const title = useSelector((state: State) => selectTitleByTimelineById(state, timelineId)); + const isDataInTimeline = useSelector((state: State) => selectDataInTimeline(state, timelineId)); + const kqlQueryObj = useSelector((state: State) => selectKqlQuery(state, timelineId)); + + const { activeTab, dataProviders, timelineType, filters, kqlMode } = useSelector((state: State) => + selectTimelineById(state, timelineId) + ); + + const combinedQueries = useMemo( + () => + combineQueries({ + config: esQueryConfig, + dataProviders, + indexPattern, + browserFields, + filters: filters ? filters : [], + kqlQuery: kqlQueryObj, + kqlMode, + }), + [browserFields, dataProviders, esQueryConfig, filters, indexPattern, kqlMode, kqlQueryObj] + ); + const isInspectDisabled = !isDataInTimeline || combinedQueries?.filterQuery === undefined; + + const closeTimeline = useCallback(() => { + createHistoryEntry(); + dispatch(timelineActions.showTimeline({ id: timelineId, show: false })); + }, [dispatch, timelineId]); + + return ( + + + + + + + + + +

{title}

+ + + + + + + + + + + + + + + + + + + {userCasesPermissions.create && userCasesPermissions.read ? ( + <> + + + + + + + + ) : null} + + + + + + + + + + + + + ); +}); + +TimelineModalHeader.displayName = 'TimelineModalHeader'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.styles.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/index.styles.tsx similarity index 84% rename from x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.styles.tsx rename to x-pack/plugins/security_solution/public/timelines/components/modal/index.styles.tsx index 95bea7d742ca10..08b3c85e2df9a1 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/pane.styles.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/index.styles.tsx @@ -19,6 +19,7 @@ import { export const usePaneStyles = () => { const EuiTheme = useEuiTheme(); const { euiTheme } = EuiTheme; + return css` // euiOverlayMask styles position: fixed; @@ -26,9 +27,6 @@ export const usePaneStyles = () => { left: 0; right: 0; bottom: 0; - display: flex; - align-items: center; - justify-content: center; background: ${transparentize(euiTheme.colors.ink, 0.5)}; z-index: ${euiTheme.levels.flyout}; @@ -36,13 +34,12 @@ export const usePaneStyles = () => { animation: ${euiAnimFadeIn} ${euiTheme.animation.fast} ease-in; } - &.timeline-wrapper--hidden { + &.timeline-portal-overlay-mask--hidden { display: none; } - .timeline-flyout { + .timeline-container { min-width: 150px; - height: inherit; position: fixed; top: var(--euiFixedHeadersOffset, 0); right: 0; @@ -53,15 +50,9 @@ export const usePaneStyles = () => { animation: ${euiAnimSlideInUp(euiTheme.size.xxl)} ${euiTheme.animation.normal} cubic-bezier(0.39, 0.575, 0.565, 1); } - - .timeline-body { - height: 100%; - display: flex; - flex-direction: column; - } } - &:not(.timeline-wrapper--full-screen) .timeline-flyout { + &:not(.timeline-portal-overlay-mask--full-screen) .timeline-container { margin: ${euiTheme.size.m}; border-radius: ${euiTheme.border.radius.medium}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/modal/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/index.test.tsx new file mode 100644 index 00000000000000..fdcdc5a501ae9c --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/index.test.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; + +import { TestProviders } from '../../../common/mock'; +import { TimelineId } from '../../../../common/types/timeline'; +import { TimelineModal } from '.'; + +jest.mock('../timeline', () => ({ + StatefulTimeline: () =>
, +})); + +const mockIsFullScreen = jest.fn(() => false); +jest.mock('../../../common/store/selectors', () => ({ + inputsSelectors: { timelineFullScreenSelector: () => mockIsFullScreen() }, +})); + +const renderTimelineModal = () => + render( + + + + ); + +describe('TimelineModal', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should render the timeline', async () => { + const { getByTestId } = renderTimelineModal(); + + expect(getByTestId('StatefulTimelineMock')).toBeInTheDocument(); + }); + + it('should render without fullscreen className', async () => { + mockIsFullScreen.mockReturnValue(false); + + const { getByTestId } = renderTimelineModal(); + + expect(getByTestId('timeline-portal-overlay-mask')).not.toHaveClass( + 'timeline-portal-overlay-mask--full-screen' + ); + }); + + it('should render with fullscreen className', async () => { + mockIsFullScreen.mockReturnValue(true); + + const { getByTestId } = renderTimelineModal(); + + expect(getByTestId('timeline-portal-overlay-mask')).toHaveClass( + 'timeline-portal-overlay-mask--full-screen' + ); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/modal/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/modal/index.tsx new file mode 100644 index 00000000000000..feb5cf74494a39 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/modal/index.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo, useRef } from 'react'; +import { i18n } from '@kbn/i18n'; +import classNames from 'classnames'; +import { StatefulTimeline } from '../timeline'; +import type { TimelineId } from '../../../../common/types/timeline'; +import { defaultRowRenderers } from '../timeline/body/renderers'; +import { DefaultCellRenderer } from '../timeline/cell_rendering/default_cell_renderer'; +import { CustomEuiPortal } from './custom_portal'; +import { useShallowEqualSelector } from '../../../common/hooks/use_selector'; +import { inputsSelectors } from '../../../common/store/selectors'; +import { usePaneStyles, OverflowHiddenGlobalStyles } from './index.styles'; + +const TIMELINE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.timeline.modal.timelinePropertiesAriaLabel', + { + defaultMessage: 'Timeline Properties', + } +); + +interface TimelineModalProps { + /** + * Id of the timeline to be displayed within the modal + */ + timelineId: TimelineId; + /** + * If true the timeline modal will be visible + */ + visible?: boolean; +} + +/** + * Renders the timeline modal. Internally this is using an EuiPortal. + */ +export const TimelineModal = React.memo(({ timelineId, visible = true }) => { + const ref = useRef(null); + const isFullScreen = useShallowEqualSelector(inputsSelectors.timelineFullScreenSelector) ?? false; + + const styles = usePaneStyles(); + const wrapperClassName = classNames('timeline-portal-overlay-mask', styles, { + 'timeline-portal-overlay-mask--full-screen': isFullScreen, + 'timeline-portal-overlay-mask--hidden': !visible, + }); + + const sibling: HTMLDivElement | null = useMemo(() => (!visible ? ref?.current : null), [visible]); + + return ( +
+ +
+
+ +
+
+ + {visible && } +
+ ); +}); + +TimelineModal.displayName = 'TimelineModal'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/translations.ts b/x-pack/plugins/security_solution/public/timelines/components/modal/translations.ts similarity index 100% rename from x-pack/plugins/security_solution/public/timelines/components/flyout/header/translations.ts rename to x-pack/plugins/security_solution/public/timelines/components/modal/translations.ts diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx index 2ae297019e9bbb..c40b54d8cd3b97 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx @@ -18,7 +18,7 @@ import { timelineDefaults } from '../../store/defaults'; import { defaultHeaders } from './body/column_headers/default_headers'; import type { CellValueElementProps } from './cell_rendering'; import { SourcererScopeName } from '../../../common/store/sourcerer/model'; -import { FlyoutHeaderPanel } from '../flyout/header'; +import { TimelineModalHeader } from '../modal/header'; import type { TimelineId, RowRenderer, TimelineTabs } from '../../../../common/types/timeline'; import { TimelineType } from '../../../../common/api/timeline'; import { useDeepEqualSelector, useShallowEqualSelector } from '../../../common/hooks/use_selector'; @@ -39,6 +39,12 @@ const TimelineTemplateBadge = styled.div` font-size: 0.8em; `; +const TimelineBody = styled.div` + height: 100%; + display: flex; + flex-direction: column; +`; + export const TimelineContext = createContext<{ timelineId: string | null }>({ timelineId: null }); export interface Props { renderCellValue: (props: CellValueElementProps) => React.ReactNode; @@ -216,7 +222,7 @@ const StatefulTimelineComponent: React.FC = ({ ref={containerElement} > -
+ {timelineType === TimelineType.template && ( {i18n.TIMELINE_TEMPLATE} @@ -227,7 +233,7 @@ const StatefulTimelineComponent: React.FC = ({ $isVisible={!timelineFullScreen} data-test-subj="timeline-hide-show-container" > - + = ({ timelineDescription={description} timelineFullScreen={timelineFullScreen} /> -
+ {showTimelineTour ? ( timelineById[timelineId] ); +/** + * Selector that returns the timeline dataProviders. + */ +const selectTimelineDataProviders = createSelector( + selectTimelineById, + (timeline) => timeline?.dataProviders +); + /** * Selector that returns the timeline saved title. */ @@ -101,9 +109,19 @@ const selectTimelineTitle = createSelector(selectTimelineById, (timeline) => tim /** * Selector that returns the timeline type. */ -const selectTimelineTimelineType = createSelector( +const selectTimelineType = createSelector(selectTimelineById, (timeline) => timeline?.timelineType); + +/** + * Selector that returns the timeline kqlQuery. + */ +const selectTimelineKqlQuery = createSelector(selectTimelineById, (timeline) => timeline?.kqlQuery); + +/** + * Selector that returns the kqlQuery.filterQuery.kuery.expression of a timeline. + */ +export const selectKqlFilterQueryExpression = createSelector( selectTimelineById, - (timeline) => timeline?.timelineType + (timeline) => timeline?.kqlQuery?.filterQuery?.kuery?.expression ); /** @@ -114,7 +132,7 @@ const selectTimelineTimelineType = createSelector( */ export const selectTitleByTimelineById = createSelector( selectTimelineTitle, - selectTimelineTimelineType, + selectTimelineType, (savedTitle, timelineType): string => { if (!isEmpty(savedTitle)) { return savedTitle; @@ -125,3 +143,30 @@ export const selectTitleByTimelineById = createSelector( return UNTITLED_TIMELINE; } ); + +/** + * Selector that returns the timeline query in a {@link Query} format. + */ +export const selectKqlQuery = createSelector( + selectTimelineDataProviders, + selectKqlFilterQueryExpression, + selectTimelineType, + (dataProviders, kqlFilterQueryExpression, timelineType): Query => { + const kqlQueryExpression = + isEmpty(dataProviders) && isEmpty(kqlFilterQueryExpression) && timelineType === 'template' + ? ' ' + : kqlFilterQueryExpression ?? ''; + return { query: kqlQueryExpression, language: 'kuery' }; + } +); + +/** + * Selector that returns true if the timeline has data providers or a kqlQuery filterQuery expression. + */ +export const selectDataInTimeline = createSelector( + selectTimelineDataProviders, + selectTimelineKqlQuery, + (dataProviders, kqlQuery): boolean => { + return !isEmpty(dataProviders) || !isEmpty(get('filterQuery.kuery.expression', kqlQuery)); + } +); diff --git a/x-pack/plugins/security_solution/public/timelines/wrapper/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/wrapper/index.test.tsx index 7eac49f7e066d2..7bf16595decdd2 100644 --- a/x-pack/plugins/security_solution/public/timelines/wrapper/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/wrapper/index.test.tsx @@ -9,7 +9,6 @@ import React from 'react'; import { render } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import '../../common/mock/react_beautiful_dnd'; - import { TestProviders } from '../../common/mock'; import { TimelineId } from '../../../common/types/timeline'; import * as timelineActions from '../store/actions'; @@ -45,7 +44,7 @@ describe('TimelineWrapper', () => { ); - expect(getByTestId('flyout-pane')).toBeInTheDocument(); + expect(getByTestId('timeline-portal-ref')).toBeInTheDocument(); expect(getByTestId('timeline-bottom-bar')).toBeInTheDocument(); }); diff --git a/x-pack/plugins/security_solution/public/timelines/wrapper/index.tsx b/x-pack/plugins/security_solution/public/timelines/wrapper/index.tsx index 7b1c0743dbce61..0a6e9599674055 100644 --- a/x-pack/plugins/security_solution/public/timelines/wrapper/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/wrapper/index.tsx @@ -9,10 +9,10 @@ import { EuiFocusTrap, EuiWindowEvent, keys } from '@elastic/eui'; import React, { useMemo, useCallback } from 'react'; import type { AppLeaveHandler } from '@kbn/core/public'; import { useDispatch } from 'react-redux'; +import { TimelineModal } from '../components/modal'; import type { TimelineId } from '../../../common/types'; import { useDeepEqualSelector } from '../../common/hooks/use_selector'; import { TimelineBottomBar } from '../components/bottom_bar'; -import { Pane } from '../components/flyout/pane'; import { getTimelineShowStatusByIdSelector } from '../store/selectors'; import { useTimelineSavePrompt } from '../../common/hooks/timeline/use_timeline_save_prompt'; import { timelineActions } from '../store'; @@ -58,7 +58,7 @@ export const TimelineWrapper: React.FC = React.memo( return ( <> - + diff --git a/x-pack/plugins/threat_intelligence/cypress/screens/timeline.ts b/x-pack/plugins/threat_intelligence/cypress/screens/timeline.ts index e9e960db9858b3..6861c63496d7d3 100644 --- a/x-pack/plugins/threat_intelligence/cypress/screens/timeline.ts +++ b/x-pack/plugins/threat_intelligence/cypress/screens/timeline.ts @@ -22,7 +22,7 @@ export const INDICATORS_TABLE_CELL_TIMELINE_BUTTON = `[data-test-subj="${CELL_TI export const TIMELINE_DATA_PROVIDERS_WRAPPER = `[data-test-subj="dataProviders"]`; export const TIMELINE_DRAGGABLE_ITEM = `[data-test-subj="providerContainer"]`; export const TIMELINE_AND_OR_BADGE = `[data-test-subj="and-or-badge"]`; -export const CLOSE_TIMELINE_BTN = '[data-test-subj="close-timeline"]'; +export const CLOSE_TIMELINE_BTN = '[data-test-subj="timeline-modal-header-close-button"]'; export const FLYOUT_OVERVIEW_TAB_TABLE_ROW_TIMELINE_BUTTON = `[data-test-subj="${INDICATORS_FLYOUT_OVERVIEW_TABLE}${VALUE_ACTION_TIMELINE_BUTTON_TEST_ID}"]`; export const FLYOUT_OVERVIEW_TAB_BLOCKS_TIMELINE_BUTTON = `[data-test-subj="${INDICATORS_FLYOUT_OVERVIEW_HIGH_LEVEL_BLOCKS}${VALUE_ACTION_TIMELINE_BUTTON_TEST_ID}"]`; export const FLYOUT_INVESTIGATE_IN_TIMELINE_ITEM = `[data-test-subj="${INDICATOR_FLYOUT_TAKE_ACTION_INVESTIGATE_IN_TIMELINE_TEST_ID}"]`; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 63b76a5cb04ac6..3d784e40249b66 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -35991,7 +35991,6 @@ "xpack.securitySolution.timeline.file.fromOriginalPathDescription": "depuis son chemin d'origine", "xpack.securitySolution.timeline.flyout.header.closeTimelineButtonLabel": "Fermer {isTimeline, select, true {la chronologie} false {le modèle}}", "xpack.securitySolution.timeline.flyout.pane.removeColumnButtonLabel": "Supprimer la colonne", - "xpack.securitySolution.timeline.flyout.pane.timelinePropertiesAriaLabel": "Propriétés de la chronologie", "xpack.securitySolution.timeline.flyoutTimelineTemplateLabel": "Modèle de chronologie", "xpack.securitySolution.timeline.fullScreenButton": "Plein écran", "xpack.securitySolution.timeline.graphOverlay.closeAnalyzerButton": "Fermer l'analyseur", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 5ac0b945b11739..4dcd88dd9f4fe3 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -35991,7 +35991,6 @@ "xpack.securitySolution.timeline.file.fromOriginalPathDescription": "元のパスから", "xpack.securitySolution.timeline.flyout.header.closeTimelineButtonLabel": "{isTimeline, select, true {タイムライン} false {テンプレート}}を閉じる", "xpack.securitySolution.timeline.flyout.pane.removeColumnButtonLabel": "列を削除", - "xpack.securitySolution.timeline.flyout.pane.timelinePropertiesAriaLabel": "タイムラインのプロパティ", "xpack.securitySolution.timeline.flyoutTimelineTemplateLabel": "タイムラインテンプレート", "xpack.securitySolution.timeline.fullScreenButton": "全画面", "xpack.securitySolution.timeline.graphOverlay.closeAnalyzerButton": "アナライザーを閉じる", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index eeb98e4c894a1f..65228022a6acf7 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -35973,7 +35973,6 @@ "xpack.securitySolution.timeline.file.fromOriginalPathDescription": "从其原始路径", "xpack.securitySolution.timeline.flyout.header.closeTimelineButtonLabel": "关闭{isTimeline, select, true {时间线} false {模板}}", "xpack.securitySolution.timeline.flyout.pane.removeColumnButtonLabel": "移除列", - "xpack.securitySolution.timeline.flyout.pane.timelinePropertiesAriaLabel": "时间线属性", "xpack.securitySolution.timeline.flyoutTimelineTemplateLabel": "时间线模板", "xpack.securitySolution.timeline.fullScreenButton": "全屏", "xpack.securitySolution.timeline.graphOverlay.closeAnalyzerButton": "关闭分析器", diff --git a/x-pack/test/security_solution_cypress/cypress/screens/security_main.ts b/x-pack/test/security_solution_cypress/cypress/screens/security_main.ts index 6eacc61dd76ad4..5836234c4058c7 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/security_main.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/security_main.ts @@ -5,8 +5,6 @@ * 2.0. */ -export const CLOSE_TIMELINE_BUTTON = '[data-test-subj="close-timeline"]'; - export const MAIN_PAGE = '[data-test-subj="kibanaChrome"]'; export const TIMELINE_TOGGLE_BUTTON = '[data-test-subj="timeline-bottom-bar-title-button"]'; diff --git a/x-pack/test/security_solution_cypress/cypress/screens/timeline.ts b/x-pack/test/security_solution_cypress/cypress/screens/timeline.ts index e348f71245662d..964cf26ec13e2f 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/timeline.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/timeline.ts @@ -26,7 +26,7 @@ export const SELECT_CASE = (id: string) => { return `[data-test-subj="cases-table-row-select-${id}"]`; }; -export const CLOSE_TIMELINE_BTN = '[data-test-subj="close-timeline"]'; +export const CLOSE_TIMELINE_BTN = '[data-test-subj="timeline-modal-header-close-button"]'; export const COMBO_BOX_INPUT = '[data-test-subj="comboBoxInput"]'; @@ -177,7 +177,7 @@ export const TIMELINE_FILTER_OPERATOR = '[data-test-subj="filterOperatorList"]'; export const TIMELINE_FILTER_VALUE = '[data-test-subj="filterParamsComboBox phraseParamsComboxBox"]'; -export const TIMELINE_FLYOUT = '[data-test-subj="timeline-flyout"]'; +export const TIMELINE_FLYOUT = '[data-test-subj="timeline-container"]'; export const TIMELINE_FLYOUT_HEADER = '[data-test-subj="query-tab-flyout-header"]'; @@ -185,7 +185,7 @@ export const TIMELINE_HEADER = '[data-test-subj="timeline-hide-show-container"]' export const TIMELINE_INSPECT_BUTTON = `${TIMELINE_FLYOUT} [data-test-subj="inspect-empty-button"]`; -export const TIMELINE_PANEL = `[data-test-subj="timeline-flyout-header-panel"]`; +export const TIMELINE_PANEL = `[data-test-subj="timeline-modal-header-panel"]`; export const TIMELINE_QUERY = '[data-test-subj="timelineQueryInput"]'; @@ -205,7 +205,7 @@ export const TIMELINE_LUCENELANGUAGE_BUTTON = '[data-test-subj="luceneLanguageMe export const TIMELINE_KQLLANGUAGE_BUTTON = '[data-test-subj="kqlLanguageMenuItem"]'; -export const TIMELINE_TITLE = '[data-test-subj="timeline-title"]'; +export const TIMELINE_TITLE = '[data-test-subj="timeline-modal-header-title"]'; export const TIMELINE_TITLE_INPUT = '[data-test-subj="save-timeline-modal-title-input"]'; @@ -225,9 +225,9 @@ export const TIMELINE_SAVE_MODAL_SAVE_AS_NEW_SWITCH = export const TIMELINE_EXIT_FULL_SCREEN_BUTTON = '[data-test-subj="exit-full-screen"]'; -export const TIMELINE_FLYOUT_WRAPPER = '[data-test-subj="flyout-pane"]'; +export const TIMELINE_FLYOUT_WRAPPER = '[data-test-subj="timeline-portal-ref"]'; -export const TIMELINE_WRAPPER = '[data-test-subj="timeline-wrapper"]'; +export const TIMELINE_WRAPPER = '[data-test-subj="timeline-portal-overlay-mask"]'; export const TIMELINE_FULL_SCREEN_BUTTON = '[data-test-subj="full-screen-active"]'; @@ -316,7 +316,7 @@ export const NEW_TIMELINE_ACTION = getDataTestSubjectSelector( 'timeline-modal-new-timeline-dropdown-button' ); -export const SAVE_TIMELINE_ACTION = getDataTestSubjectSelector('save-timeline-action'); +export const SAVE_TIMELINE_ACTION = getDataTestSubjectSelector('timeline-modal-save-timeline'); export const SAVE_TIMELINE_ACTION_BTN = getDataTestSubjectSelector('timeline-modal-save-timeline'); export const SAVE_TIMELINE_TOOLTIP = getDataTestSubjectSelector( diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/security_main.ts b/x-pack/test/security_solution_cypress/cypress/tasks/security_main.ts index fbf31d19e5a4c0..3ed801310da1cf 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/security_main.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/security_main.ts @@ -6,8 +6,9 @@ */ import { recurse } from 'cypress-recurse'; -import { CLOSE_TIMELINE_BUTTON, TIMELINE_BOTTOM_BAR_TOGGLE_BUTTON } from '../screens/security_main'; +import { TIMELINE_BOTTOM_BAR_TOGGLE_BUTTON } from '../screens/security_main'; import { + CLOSE_TIMELINE_BTN, TIMELINE_EXIT_FULL_SCREEN_BUTTON, TIMELINE_FULL_SCREEN_BUTTON, TIMELINE_WRAPPER, @@ -20,12 +21,12 @@ export const openTimelineUsingToggle = () => { return cy.get(TIMELINE_WRAPPER); }, // Retry if somehow the timeline wrapper is still hidden - ($timelineWrapper) => !$timelineWrapper.hasClass('timeline-wrapper--hidden') + ($timelineWrapper) => !$timelineWrapper.hasClass('timeline-portal-overlay-mask--hidden') ); }; export const closeTimelineUsingCloseButton = () => { - cy.get(CLOSE_TIMELINE_BUTTON).filter(':visible').click(); + cy.get(CLOSE_TIMELINE_BTN).filter(':visible').click(); }; export const enterFullScreenMode = () => { diff --git a/x-pack/test/security_solution_ftr/page_objects/timeline/index.ts b/x-pack/test/security_solution_ftr/page_objects/timeline/index.ts index 26345722ef3268..2e5cc8492eaa2a 100644 --- a/x-pack/test/security_solution_ftr/page_objects/timeline/index.ts +++ b/x-pack/test/security_solution_ftr/page_objects/timeline/index.ts @@ -10,7 +10,7 @@ import { DATE_RANGE_OPTION_TO_TEST_SUBJ_MAP } from '@kbn/security-solution-plugi import { FtrService } from '../../../functional/ftr_provider_context'; const TIMELINE_BOTTOM_BAR_CONTAINER_TEST_SUBJ = 'timeline-bottom-bar'; -const TIMELINE_CLOSE_BUTTON_TEST_SUBJ = 'close-timeline'; +const TIMELINE_CLOSE_BUTTON_TEST_SUBJ = 'timeline-modal-header-close-button'; const TIMELINE_MODAL_PAGE_TEST_SUBJ = 'timeline'; const TIMELINE_TAB_QUERY_TEST_SUBJ = 'timeline-tab-content-query'; From 42b42c8fa53e85895aad876ed57ce398b36ed76a Mon Sep 17 00:00:00 2001 From: Marta Bondyra <4283304+mbondyra@users.noreply.github.com> Date: Wed, 31 Jan 2024 22:35:01 +0100 Subject: [PATCH 19/20] [Lens] [A11y] fix creating or removing layers in Lens looses focus (#175893) ## Summary Fixes https://github.com/elastic/kibana/issues/175191 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4989 I replaced the unit tests with functional ones because of two reasons : first, this is a complex behavior and it's tricky to test it with unit tests with confidence. Secondly, it was actually tested but yet tests were passing and the bug happened anyway. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../config_panel/config_panel.test.tsx | 91 ------------------- .../config_panel/layer_panel.test.tsx | 2 + .../editor_frame/config_panel/layer_panel.tsx | 9 +- .../config_panel/use_focus_update.tsx | 2 +- x-pack/test/accessibility/apps/group2/lens.ts | 28 ++++++ 5 files changed, 38 insertions(+), 94 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx index 04d69c1afc5715..617b15b43b43df 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx @@ -27,7 +27,6 @@ import { generateId } from '../../../id_generator'; import { mountWithProvider } from '../../../mocks'; import { LayerTypes } from '@kbn/expression-xy-plugin/public'; import { ReactWrapper } from 'enzyme'; -import { addLayer } from '../../../state_management'; import { createIndexPatternServiceMock } from '../../../mocks/data_views_service_mock'; import { AddLayerButton } from '../../../visualizations/xy/add_layer'; import { LayerType } from '@kbn/visualizations-plugin/common'; @@ -190,96 +189,6 @@ describe('ConfigPanel', () => { ); }); - describe('focus behavior when adding or removing layers', () => { - it('should focus the only layer when resetting the layer', async () => { - const { instance } = await prepareAndMountComponent(getDefaultProps()); - const firstLayerFocusable = instance - .find(LayerPanel) - .first() - .find('section') - .first() - .instance(); - act(() => { - instance.find('[data-test-subj="lnsLayerRemove--0"]').first().simulate('click'); - }); - instance.update(); - - const focusedEl = document.activeElement; - expect(focusedEl).toEqual(firstLayerFocusable); - }); - - it('should focus the second layer when removing the first layer', async () => { - const datasourceMap = mockDatasourceMap(); - const defaultProps = getDefaultProps({ datasourceMap }); - // overwriting datasourceLayers to test two layers - frame.datasourceLayers = { - first: datasourceMap.testDatasource.publicAPIMock, - second: datasourceMap.testDatasource.publicAPIMock, - }; - - const { instance } = await prepareAndMountComponent(defaultProps); - const secondLayerFocusable = instance - .find(LayerPanel) - .at(1) - .find('section') - .first() - .instance(); - act(() => { - instance.find('[data-test-subj="lnsLayerRemove--0"]').first().simulate('click'); - }); - instance.update(); - - const focusedEl = document.activeElement; - expect(focusedEl).toEqual(secondLayerFocusable); - }); - - it('should focus the first layer when removing the second layer', async () => { - const datasourceMap = mockDatasourceMap(); - const defaultProps = getDefaultProps({ datasourceMap }); - // overwriting datasourceLayers to test two layers - frame.datasourceLayers = { - first: datasourceMap.testDatasource.publicAPIMock, - second: datasourceMap.testDatasource.publicAPIMock, - }; - const { instance } = await prepareAndMountComponent(defaultProps); - const firstLayerFocusable = instance - .find(LayerPanel) - .first() - .find('section') - .first() - .instance(); - act(() => { - instance.find('[data-test-subj="lnsLayerRemove--1"]').first().simulate('click'); - }); - instance.update(); - - const focusedEl = document.activeElement; - expect(focusedEl).toEqual(firstLayerFocusable); - }); - - it('should focus the added layer', async () => { - const datasourceMap = mockDatasourceMap(); - frame.datasourceLayers = { - first: datasourceMap.testDatasource.publicAPIMock, - newId: datasourceMap.testDatasource.publicAPIMock, - }; - - const defaultProps = getDefaultProps({ datasourceMap }); - - const { instance } = await prepareAndMountComponent(defaultProps, { - dispatch: jest.fn((x) => { - if (x.type === addLayer.type) { - frame.datasourceLayers.newId = datasourceMap.testDatasource.publicAPIMock; - } - }), - }); - - addNewLayer(instance); - const focusedEl = document.activeElement; - expect(focusedEl?.children[0].getAttribute('data-test-subj')).toEqual('lns-layerPanel-1'); - }); - }); - describe('initial default value', () => { function clickToAddDimension(instance: ReactWrapper) { act(() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index cef598de31af0d..c53198957bb0ee 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -950,6 +950,7 @@ describe('LayerPanel', () => { }); it('should reorder when dropping in the same group', async () => { + jest.useFakeTimers(); mockVisualization.getConfiguration.mockReturnValue({ groups: [ { @@ -997,6 +998,7 @@ describe('LayerPanel', () => { .find('[data-test-subj="lnsDragDrop-keyboardHandler"]') .at(1) .instance(); + jest.runAllTimers(); const focusedEl = document.activeElement; expect(focusedEl).toEqual(secondButton); }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 024fb04998d378..6680dc984beed7 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -363,8 +363,13 @@ export function LayerPanel( return ( <> -
- +
+
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/use_focus_update.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/use_focus_update.tsx index e7ee06a020ece9..7065c37c1daf2b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/use_focus_update.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/use_focus_update.tsx @@ -30,7 +30,7 @@ export function useFocusUpdate(ids: string[]) { const element = nextFocusedId && refsById.get(nextFocusedId); if (element) { const focusable = getFirstFocusable(element); - focusable?.focus(); + setTimeout(() => focusable?.focus()); setNextFocusedId(null); } }, [ids, refsById, nextFocusedId]); diff --git a/x-pack/test/accessibility/apps/group2/lens.ts b/x-pack/test/accessibility/apps/group2/lens.ts index 860138fc777012..e4ef53efc3548a 100644 --- a/x-pack/test/accessibility/apps/group2/lens.ts +++ b/x-pack/test/accessibility/apps/group2/lens.ts @@ -5,6 +5,7 @@ * 2.0. */ +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { @@ -14,6 +15,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const listingTable = getService('listingTable'); const kibanaServer = getService('kibanaServer'); + const find = getService('find'); + + const hasFocus = async (testSubject: string) => { + const targetElement = await testSubjects.find(testSubject); + const activeElement = await find.activeElement(); + return (await targetElement._webElement.getId()) === (await activeElement._webElement.getId()); + }; describe('Lens Accessibility', () => { const lensChartName = 'MyLensChart'; @@ -175,5 +183,25 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await listingTable.clickDeleteSelected(); await PageObjects.common.clickConfirmOnModal(); }); + + describe('focus behavior when adding or removing layers', () => { + it('should focus the added layer', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.createLayer(); + expect(await hasFocus('lns-layerPanel-1')).to.be(true); + }); + it('should focus the remaining layer when the first is removed', async () => { + await PageObjects.lens.removeLayer(0); + expect(await hasFocus('lns-layerPanel-0')).to.be(true); + await PageObjects.lens.createLayer(); + await PageObjects.lens.removeLayer(1); + expect(await hasFocus('lns-layerPanel-0')).to.be(true); + }); + it('should focus the only layer when resetting the layer', async () => { + await PageObjects.lens.removeLayer(); + expect(await hasFocus('lns-layerPanel-0')).to.be(true); + }); + }); }); } From cb8637a7d80dbee8e0ed82c01cc90654e632b19d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 31 Jan 2024 23:53:56 +0100 Subject: [PATCH 20/20] [ML] AIOps: Fix functional tests (#174361) ## Summary Fixes #172606 - Instead of creating the Kibana Sample Logs Data via UI, this uses now `esArchiver` to restore `kibana_sample_data_logs_tsdb`. Restoring via UI caused some flakyness, because in rare cases additional results could pop up, probably because of some randomization that results in certain patterns sometimes. - Adds a `retry` to assert the group results table - Before asserting the result tables, scroll the table into view so in case of errors the screenshots show the results. - Fixes an issue with timezones for the dynamically generated test datasets. If a local dev setup was not set to UTC the test assertions would fail. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../apps/aiops/log_rate_analysis.ts | 126 ++++++++---------- .../kibana_logs_data_view_test_data.ts | 4 +- .../aiops/log_rate_analysis_data_generator.ts | 12 +- .../services/aiops/log_rate_analysis_page.ts | 3 +- .../log_rate_analysis_results_groups_table.ts | 4 + 5 files changed, 73 insertions(+), 76 deletions(-) diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts index 4bb1393b5c910b..45fef76fa71706 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts @@ -18,6 +18,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const browser = getService('browser'); const elasticChart = getService('elasticChart'); const aiops = getService('aiops'); + const retry = getService('retry'); // AIOps / Log Rate Analysis lives in the ML UI so we need some related services. const ml = getService('ml'); @@ -165,45 +166,52 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { // The group switch should be disabled by default await aiops.logRateAnalysisPage.assertLogRateAnalysisResultsGroupSwitchExists(false); - if (!isTestDataExpectedWithSampleProbability(testData.expected)) { - // Enabled grouping - await aiops.logRateAnalysisPage.clickLogRateAnalysisResultsGroupSwitchOn(); + await retry.tryForTime(30 * 1000, async () => { + if (!isTestDataExpectedWithSampleProbability(testData.expected)) { + // Enabled grouping + await aiops.logRateAnalysisPage.clickLogRateAnalysisResultsGroupSwitchOn(); - await aiops.logRateAnalysisResultsGroupsTable.assertLogRateAnalysisResultsTableExists(); + await aiops.logRateAnalysisResultsGroupsTable.assertLogRateAnalysisResultsTableExists(); + await aiops.logRateAnalysisResultsGroupsTable.scrollAnalysisTableIntoView(); - const analysisGroupsTable = - await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); + const analysisGroupsTable = + await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - const actualAnalysisGroupsTable = orderBy(analysisGroupsTable, 'group'); - const expectedAnalysisGroupsTable = orderBy(testData.expected.analysisGroupsTable, 'group'); + const actualAnalysisGroupsTable = orderBy(analysisGroupsTable, 'group'); + const expectedAnalysisGroupsTable = orderBy( + testData.expected.analysisGroupsTable, + 'group' + ); - expect(actualAnalysisGroupsTable).to.be.eql( - expectedAnalysisGroupsTable, - `Expected analysis groups table to be ${JSON.stringify( - expectedAnalysisGroupsTable - )}, got ${JSON.stringify(actualAnalysisGroupsTable)}` - ); + expect(actualAnalysisGroupsTable).to.be.eql( + expectedAnalysisGroupsTable, + `Expected analysis groups table to be ${JSON.stringify( + expectedAnalysisGroupsTable + )}, got ${JSON.stringify(actualAnalysisGroupsTable)}` + ); + } + }); + if (!isTestDataExpectedWithSampleProbability(testData.expected)) { await ml.testExecution.logTestStep('expand table row'); await aiops.logRateAnalysisResultsGroupsTable.assertExpandRowButtonExists(); await aiops.logRateAnalysisResultsGroupsTable.expandRow(); + await aiops.logRateAnalysisResultsGroupsTable.scrollAnalysisTableIntoView(); - if (!isTestDataExpectedWithSampleProbability(testData.expected)) { - const analysisTable = await aiops.logRateAnalysisResultsTable.parseAnalysisTable(); - - const actualAnalysisTable = orderBy(analysisTable, ['fieldName', 'fieldValue']); - const expectedAnalysisTable = orderBy(testData.expected.analysisTable, [ - 'fieldName', - 'fieldValue', - ]); - - expect(actualAnalysisTable).to.be.eql( - expectedAnalysisTable, - `Expected analysis table results to be ${JSON.stringify( - expectedAnalysisTable - )}, got ${JSON.stringify(actualAnalysisTable)}` - ); - } + const analysisTable = await aiops.logRateAnalysisResultsTable.parseAnalysisTable(); + + const actualAnalysisTable = orderBy(analysisTable, ['fieldName', 'fieldValue']); + const expectedAnalysisTable = orderBy(testData.expected.analysisTable, [ + 'fieldName', + 'fieldValue', + ]); + + expect(actualAnalysisTable).to.be.eql( + expectedAnalysisTable, + `Expected analysis table results to be ${JSON.stringify( + expectedAnalysisTable + )}, got ${JSON.stringify(actualAnalysisTable)}` + ); await ml.testExecution.logTestStep('open the field filter'); await aiops.logRateAnalysisPage.assertFieldFilterPopoverButtonExists(false); @@ -226,23 +234,21 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await ml.testExecution.logTestStep('regroup results'); await aiops.logRateAnalysisPage.clickFieldFilterApplyButton(); - if (!isTestDataExpectedWithSampleProbability(testData.expected)) { - const filteredAnalysisGroupsTable = - await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - - const actualFilteredAnalysisGroupsTable = orderBy(filteredAnalysisGroupsTable, 'group'); - const expectedFilteredAnalysisGroupsTable = orderBy( - testData.expected.filteredAnalysisGroupsTable, - 'group' - ); - - expect(actualFilteredAnalysisGroupsTable).to.be.eql( - expectedFilteredAnalysisGroupsTable, - `Expected filtered analysis groups table to be ${JSON.stringify( - expectedFilteredAnalysisGroupsTable - )}, got ${JSON.stringify(actualFilteredAnalysisGroupsTable)}` - ); - } + const filteredAnalysisGroupsTable = + await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); + + const actualFilteredAnalysisGroupsTable = orderBy(filteredAnalysisGroupsTable, 'group'); + const expectedFilteredAnalysisGroupsTable = orderBy( + testData.expected.filteredAnalysisGroupsTable, + 'group' + ); + + expect(actualFilteredAnalysisGroupsTable).to.be.eql( + expectedFilteredAnalysisGroupsTable, + `Expected filtered analysis groups table to be ${JSON.stringify( + expectedFilteredAnalysisGroupsTable + )}, got ${JSON.stringify(actualFilteredAnalysisGroupsTable)}` + ); } if (testData.action !== undefined) { @@ -268,41 +274,23 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); } - // Failing: See https://github.com/elastic/kibana/issues/172606 - describe.skip('log rate analysis', async function () { + describe('log rate analysis', async function () { for (const testData of logRateAnalysisTestData) { describe(`with '${testData.sourceIndexOrSavedSearch}'`, function () { before(async () => { await aiops.logRateAnalysisDataGenerator.generateData(testData.dataGenerator); + await ml.testResources.setKibanaTimeZoneToUTC(); + + await ml.securityUI.loginAsMlPowerUser(); await ml.testResources.createDataViewIfNeeded( testData.sourceIndexOrSavedSearch, '@timestamp' ); - - await ml.testResources.setKibanaTimeZoneToUTC(); - - if (testData.dataGenerator === 'kibana_sample_data_logs') { - await PageObjects.security.login('elastic', 'changeme', { - expectSuccess: true, - }); - - await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', { - useActualUrl: true, - }); - await PageObjects.header.waitUntilLoadingHasFinished(); - await PageObjects.home.addSampleDataSet('logs'); - await PageObjects.header.waitUntilLoadingHasFinished(); - } else { - await ml.securityUI.loginAsMlPowerUser(); - } }); after(async () => { await elasticChart.setNewChartUiDebugFlag(false); - if (testData.dataGenerator !== 'kibana_sample_data_logs') { - await ml.testResources.deleteDataViewByTitle(testData.sourceIndexOrSavedSearch); - } await aiops.logRateAnalysisDataGenerator.removeGeneratedData(testData.dataGenerator); }); diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts index 2d85fe1e64210b..a22015a6235597 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts @@ -14,7 +14,7 @@ export const kibanaLogsDataViewTestData: TestData = { analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, dataGenerator: 'kibana_sample_data_logs', isSavedSearch: false, - sourceIndexOrSavedSearch: 'kibana_sample_data_logs', + sourceIndexOrSavedSearch: 'kibana_sample_data_logstsdb', brushIntervalFactor: 1, chartClickCoordinates: [235, 0], fieldSelectorSearch: 'referer', @@ -29,7 +29,7 @@ export const kibanaLogsDataViewTestData: TestData = { }, }, expected: { - totalDocCountFormatted: '14,074', + totalDocCountFormatted: '14,068', analysisGroupsTable: [ { group: diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index 7e1ead80b4ff83..824e296d448d32 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -46,7 +46,9 @@ const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; function getTextFieldMessage(timestamp: number, user: string, url: string, responseCode: string) { const date = new Date(timestamp); - return `${user} [${date.toLocaleString('en-US')}] "GET /${url} HTTP/1.1" ${responseCode}`; + return `${user} [${date.toLocaleString('en-US', { + timeZone: 'UTC', + })}] "GET /${url} HTTP/1.1" ${responseCode}`; } function getArtificialLogsWithDeviation( @@ -224,7 +226,9 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider public async generateData(dataGenerator: LogRateAnalysisDataGenerator) { switch (dataGenerator) { case 'kibana_sample_data_logs': - // will be added via UI + await esArchiver.loadIfNeeded( + 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_tsdb' + ); break; case 'farequote_with_spike': @@ -328,7 +332,9 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider public async removeGeneratedData(dataGenerator: LogRateAnalysisDataGenerator) { switch (dataGenerator) { case 'kibana_sample_data_logs': - // do not remove + await esArchiver.unload( + 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_tsdb' + ); break; case 'farequote_with_spike': diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts index a8733fb2114a70..6b1b45f4aa9ebe 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts @@ -152,9 +152,8 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr }, async clickLogRateAnalysisResultsGroupSwitchOn() { - await testSubjects.clickWhenNotDisabledWithoutRetry('aiopsLogRateAnalysisGroupSwitchOn'); - await retry.tryForTime(30 * 1000, async () => { + await testSubjects.clickWhenNotDisabledWithoutRetry('aiopsLogRateAnalysisGroupSwitchOn'); await testSubjects.existOrFail('aiopsLogRateAnalysisGroupSwitch checked'); }); }, diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_results_groups_table.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_results_groups_table.ts index 129778a23d69a6..abe734ef6526f4 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_results_groups_table.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_results_groups_table.ts @@ -59,6 +59,10 @@ export function LogRateAnalysisResultsGroupsTableProvider({ getService }: FtrPro return rows; } + public async scrollAnalysisTableIntoView() { + await testSubjects.scrollIntoView('aiopsLogRateAnalysisResultsGroupsTable'); + } + public rowSelector(rowId: string, subSelector?: string) { const row = `~aiopsLogRateAnalysisResultsGroupsTable > ~row-${rowId}`; return !subSelector ? row : `${row} > ${subSelector}`;