-
Notifications
You must be signed in to change notification settings - Fork 8.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Siem query rule - reduce field_caps usage (#184890)
## Summary Previously, the siem query rule loaded the full set of fields for an index pattern when running a query. This could load 5k fields or more. Now it only loads the fields necessary for the query. Changes as part of this PR - The data plugin exports `queryToFields` which takes a query and returns a list of the fields required to translate the query to ES DSL. - `queryToFields` properly handles all filter types, previously expected unified search bar provided filters. - `createSecurityRuleTypeWrapper` has been modified to skip field loading for the siem query rule - `getFilter` takes an optional `loadFields` arguments which loads only necessary fields - `getQueryFilterLoadFields` was created - based on `getQueryFilter` but also loads necessary fields
- Loading branch information
Showing
8 changed files
with
152 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
src/plugins/data/common/search/search_source/query_to_fields.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| import { DataViewLazy } from '@kbn/data-views-plugin/common'; | ||
| import { fromKueryExpression, getKqlFieldNames } from '@kbn/es-query'; | ||
| import type { SearchRequest } from './fetch'; | ||
| import { EsQuerySortValue } from '../..'; | ||
|
|
||
| export async function queryToFields({ | ||
| dataView, | ||
| sort, | ||
| request, | ||
| }: { | ||
| dataView: DataViewLazy; | ||
| sort?: EsQuerySortValue | EsQuerySortValue[]; | ||
| request: SearchRequest; | ||
| }) { | ||
| let fields = dataView.timeFieldName ? [dataView.timeFieldName] : []; | ||
| if (sort) { | ||
| const sortArr = Array.isArray(sort) ? sort : [sort]; | ||
| fields.push(...sortArr.flatMap((s) => Object.keys(s))); | ||
| } | ||
| for (const query of request.query) { | ||
| if (query.query) { | ||
| const nodes = fromKueryExpression(query.query); | ||
| const queryFields = getKqlFieldNames(nodes); | ||
| fields = fields.concat(queryFields); | ||
| } | ||
| } | ||
| const filters = request.filters; | ||
| if (filters) { | ||
| const filtersArr = Array.isArray(filters) ? filters : [filters]; | ||
| for (const f of filtersArr) { | ||
| // unified search bar filters have meta object and key (regular filters) | ||
| // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) | ||
| // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) | ||
| if (f?.meta?.key && f.meta.disabled !== true) { | ||
| fields.push(f.meta.key); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // if source filtering is enabled, we need to fetch all the fields | ||
| const fieldName = | ||
| dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length ? ['*'] : fields; | ||
|
|
||
| if (fieldName.length) { | ||
| return (await dataView.getFields({ fieldName })).getFieldMapSorted(); | ||
| } | ||
|
|
||
| // no fields needed to be loaded for query | ||
| return {}; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
...ity_solution/server/lib/detection_engine/rule_types/utils/get_query_filter_load_fields.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import type { Language } from '@kbn/securitysolution-io-ts-alerting-types'; | ||
| import type { Filter, EsQueryConfig, DataViewFieldBase } from '@kbn/es-query'; | ||
| import { DataView } from '@kbn/data-views-plugin/server'; | ||
| import { queryToFields } from '@kbn/data-plugin/common'; | ||
| import type { DataViewsContract } from '@kbn/data-views-plugin/common'; | ||
| import type { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common'; | ||
| import { buildEsQuery } from '@kbn/es-query'; | ||
| import type { ESBoolQuery } from '../../../../../common/typed_json'; | ||
| import { getAllFilters } from './get_query_filter'; | ||
| import type { | ||
| IndexPatternArray, | ||
| RuleQuery, | ||
| } from '../../../../../common/api/detection_engine/model/rule_schema'; | ||
|
|
||
| export const getQueryFilterLoadFields = | ||
| (dataViewsService: DataViewsContract) => | ||
| async ({ | ||
| query, | ||
| language, | ||
| filters, | ||
| index, | ||
| exceptionFilter, | ||
| }: { | ||
| query: RuleQuery; | ||
| language: Language; | ||
| filters: unknown; | ||
| index: IndexPatternArray; | ||
| exceptionFilter: Filter | undefined; | ||
| fields?: DataViewFieldBase[]; | ||
| }): Promise<ESBoolQuery> => { | ||
| const config: EsQueryConfig = { | ||
| allowLeadingWildcards: true, | ||
| queryStringOptions: { analyze_wildcard: true }, | ||
| ignoreFilterIfFieldNotInIndex: false, | ||
| dateFormatTZ: 'Zulu', | ||
| }; | ||
|
|
||
| const initialQuery = { query, language }; | ||
| const allFilters = getAllFilters(filters as Filter[], exceptionFilter); | ||
|
|
||
| const title = (index ?? []).join(); | ||
|
|
||
| const dataViewLazy = await dataViewsService.createDataViewLazy({ title }); | ||
|
|
||
| const flds = await queryToFields({ | ||
| dataView: dataViewLazy, | ||
| request: { query: [initialQuery], filters: allFilters }, | ||
| }); | ||
|
|
||
| const dataViewLimitedFields = new DataView({ | ||
| spec: { title }, | ||
| fieldFormats: {} as unknown as FieldFormatsStartCommon, | ||
| shortDotsEnable: false, | ||
| metaFields: [], | ||
| }); | ||
|
|
||
| dataViewLimitedFields.fields.replaceAll(Object.values(flds).map((fld) => fld.toSpec())); | ||
|
|
||
| return buildEsQuery(dataViewLimitedFields, initialQuery, allFilters, config); | ||
| }; |