From 3393d87959c29349c02ff6557fe41b13a44a95d4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 23 Aug 2023 13:12:46 -0600 Subject: [PATCH] Add support for geo_shape fields as the entity geospatial field when creating tracking containment alerts (#164100) Closes https://github.com/elastic/kibana/issues/163996 ### To test 1) Checkout [fake tracks geo_shape branch](https://github.com/nreese/faketracks/tree/geo_shape) 2) run npm install 3) run `node ./generate_tracks.js` 4) in kibana, create `tracks*` data view 5) create map, use "create index" and draw boundaries that intersect tracks. See screen shot Screen Shot 2023-08-17 at 2 49 52 PM 6) create geo containment alert where entity index is `tracks*` and boundaries index is `boundaries`. 7) Verify alerts get generated with entity geo_shape locations --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../rule-types/geo-rule-types.asciidoc | 9 +- ...inment_alert_type_expression.test.tsx.snap | 6 +- .../expressions/entity_index_expression.tsx | 2 +- .../rule_types/geo_containment/readme.md | 2 +- .../rule_types/geo_containment/types.ts | 3 +- .../geo_containment/executor.test.ts | 6 +- .../geo_containment/lib/alert_context.test.ts | 31 ++++++- .../geo_containment/lib/alert_context.ts | 5 +- .../geo_containment/lib/es_query_builder.ts | 7 +- .../get_entities_and_generate_alerts.test.ts | 17 ++++ .../lib/get_entities_and_generate_alerts.ts | 5 + .../lib/transform_results.test.ts | 31 ++++--- .../geo_containment/lib/transform_results.ts | 93 +++++++++---------- .../tests/es_sample_response.json | 10 +- .../es_sample_response_with_nesting.json | 8 +- .../rule_types/geo_containment/types.ts | 4 + 16 files changed, 148 insertions(+), 91 deletions(-) diff --git a/docs/user/alerting/rule-types/geo-rule-types.asciidoc b/docs/user/alerting/rule-types/geo-rule-types.asciidoc index 837d7761a2e063..cfed5152e7657f 100644 --- a/docs/user/alerting/rule-types/geo-rule-types.asciidoc +++ b/docs/user/alerting/rule-types/geo-rule-types.asciidoc @@ -10,7 +10,7 @@ In the event that an entity is contained within a boundary, an alert may be gene === Requirements To create a tracking containment rule, the following requirements must be present: -- *Tracks index or data view*: An index containing a `geo_point` field, `date` field, +- *Tracks index or data view*: An index containing a `geo_point` or `geo_shape` field, `date` field, and some form of entity identifier. An entity identifier is a `keyword` or `number` field that consistently identifies the entity to be tracked. The data in this index should be dynamically updating so that there are entity movements to alert upon. @@ -36,12 +36,7 @@ as well as two Kuery bars used to provide additional filtering context for each image::user/alerting/images/alert-types-tracking-containment-conditions.png[Define the condition to detect,width=75%] // NOTE: This is an autogenerated screenshot. Do not edit it directly. -Index (entity):: This clause requires an *index or data view*, a *time field* that will be used for the *time window*, and a *`geo_point` field* for tracking. -When entity:: This clause specifies which crossing option to track. The values -*Entered*, *Exited*, and *Crossed* can be selected to indicate which crossing conditions -should trigger a rule. *Entered* alerts on entry into a boundary, *Exited* alerts on exit -from a boundary, and *Crossed* alerts on all boundary crossings whether they be entrances -or exits. +Index (entity):: This clause requires an *index or data view*, a *time field* that will be used for the *time window*, and a *`geo_point` or `geo_shape` field* for tracking. Index (Boundary):: This clause requires an *index or data view*, a *`geo_shape` field* identifying boundaries, and an optional *Human-readable boundary name* for better alerting messages. diff --git a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/query_builder/__snapshots__/geo_containment_alert_type_expression.test.tsx.snap b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/query_builder/__snapshots__/geo_containment_alert_type_expression.test.tsx.snap index 8036c63e4f48ed..abef05ed9b3438 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/query_builder/__snapshots__/geo_containment_alert_type_expression.test.tsx.snap +++ b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/query_builder/__snapshots__/geo_containment_alert_type_expression.test.tsx.snap @@ -89,7 +89,7 @@ exports[`should render BoundaryIndexExpression 1`] = ` exports[`should render EntityIndexExpression 1`] = ` = ({ isInvalid={isInvalid} value={indexPattern.title} defaultValue={i18n.translate('xpack.stackAlerts.geoContainment.entityIndexSelect', { - defaultMessage: 'Select a data view and geo point field', + defaultMessage: 'Select a data view and geospatial field', })} popoverContent={indexPopover} expressionDescription={i18n.translate('xpack.stackAlerts.geoContainment.entityIndexLabel', { diff --git a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/readme.md b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/readme.md index 4b23f95a6e2e0c..0ee0c19f0d4321 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/readme.md +++ b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/readme.md @@ -4,7 +4,7 @@ There are several steps required to set up geo containment alerts for testing in that allows you to view triggered alerts as they happen. These instructions outline how to load test data, but really these steps can be used to load any data for geo containment alerts so long as you have the following data: -- An index containing a`geo_point` field and a `date` field. This data is presumed to +- An index containing a`geo_point` or `geo_shape` field and a `date` field. This data is presumed to be dynamic (updated). - An index containing `geo_shape` data, such as boundary data, bounding box data, etc. This data is presumed to be static (not updated). Shape data matching the query is diff --git a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/types.ts b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/types.ts index 949ebe6ff1e393..b34dd9ec4f8d27 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/types.ts +++ b/x-pack/plugins/stack_alerts/public/rule_types/geo_containment/types.ts @@ -23,6 +23,5 @@ export interface GeoContainmentAlertParams extends RuleTypeParams { boundaryIndexQuery?: Query; } -// Will eventually include 'geo_shape' -export const ES_GEO_FIELD_TYPES = ['geo_point']; +export const ES_GEO_FIELD_TYPES = ['geo_point', 'geo_shape']; export const ES_GEO_SHAPE_TYPES = ['geo_shape']; diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/executor.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/executor.test.ts index 302389bb8d8d00..cec83d2cf9e714 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/executor.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/executor.test.ts @@ -220,7 +220,8 @@ describe('getGeoContainmentExecutor', () => { { dateInShape: '2021-04-28T16:56:11.923Z', docId: 'ZVBoGXkBsFLYN2Tj1wmV', - location: [-73.99018926545978, 40.751759740523994], + location: [0, 0], + locationWkt: 'POINT (-73.99018926545978 40.751759740523994)', shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk', }, ], @@ -228,7 +229,8 @@ describe('getGeoContainmentExecutor', () => { { dateInShape: '2021-04-28T16:56:11.923Z', docId: 'ZlBoGXkBsFLYN2Tj1wmV', - location: [-73.99561604484916, 40.75449890457094], + location: [0, 0], + locationWkt: 'POINT (-73.99561604484916 40.75449890457094)', shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk', }, ], diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.test.ts index 77476e0a0ca9dc..20649dd3d0ce42 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.test.ts @@ -9,6 +9,31 @@ import { getContainedAlertContext, getRecoveredAlertContext } from './alert_cont import { OTHER_CATEGORY } from '../constants'; test('getContainedAlertContext', () => { + expect( + getContainedAlertContext({ + entityName: 'entity1', + containment: { + location: [0, 0], + locationWkt: 'POINT (100 0)', + shapeLocationId: 'boundary1Id', + dateInShape: '2022-06-21T16:56:11.923Z', + docId: 'docId', + }, + shapesIdsNamesMap: { boundary1Id: 'boundary1Name' }, + windowEnd: new Date('2022-06-21T17:00:00.000Z'), + }) + ).toEqual({ + containingBoundaryId: 'boundary1Id', + containingBoundaryName: 'boundary1Name', + detectionDateTime: '2022-06-21T17:00:00.000Z', + entityDateTime: '2022-06-21T16:56:11.923Z', + entityDocumentId: 'docId', + entityId: 'entity1', + entityLocation: 'POINT (100 0)', + }); +}); + +test('getContainedAlertContext for backwards compatible number[] location format', () => { expect( getContainedAlertContext({ entityName: 'entity1', @@ -37,7 +62,8 @@ describe('getRecoveredAlertContext', () => { const activeEntities = new Map(); activeEntities.set('entity1', [ { - location: [100, 0], + location: [0, 0], + locationWkt: 'POINT (100 0)', shapeLocationId: 'boundary1Id', dateInShape: '2022-06-21T16:56:11.923Z', docId: 'docId', @@ -63,7 +89,8 @@ describe('getRecoveredAlertContext', () => { const inactiveEntities = new Map(); inactiveEntities.set('entity1', [ { - location: [100, 0], + location: [0, 0], + locationWkt: 'POINT (100 0)', shapeLocationId: OTHER_CATEGORY, dateInShape: '2022-06-21T16:56:11.923Z', docId: 'docId', diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.ts index 42e65edaec789c..6550a681e0321d 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/alert_context.ts @@ -50,7 +50,10 @@ function getAlertContext({ entityId: entityName, entityDateTime: containment.dateInShape || null, entityDocumentId: containment.docId, - entityLocation: `POINT (${containment.location[0]} ${containment.location[1]})`, + entityLocation: + containment.locationWkt !== undefined + ? containment.locationWkt + : `POINT (${containment.location[0]} ${containment.location[1]})`, detectionDateTime: new Date(windowEnd).toISOString(), }; if (!isRecovered) { diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/es_query_builder.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/es_query_builder.ts index 44d453401078c4..4927f52da78dda 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/es_query_builder.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/es_query_builder.ts @@ -63,13 +63,16 @@ export async function executeEsQuery( }, }, ], - docvalue_fields: [ + fields: [ entity, { field: dateField, format: 'strict_date_optional_time', }, - geoField, + { + field: geoField, + format: 'wkt', + }, ], _source: false, }, diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.test.ts index 7433492d3aacc3..339e53b03056bc 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.test.ts @@ -54,6 +54,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '123', dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', @@ -65,6 +66,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '456', dateInShape: 'Wed Dec 16 2020 15:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId2', @@ -76,6 +78,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '789', dateInShape: 'Wed Dec 23 2020 16:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId3', @@ -141,6 +144,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '999', dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId7', @@ -166,6 +170,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '999', dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId7', @@ -204,6 +209,7 @@ describe('getEntitiesAndGenerateAlerts', () => { const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: OTHER_CATEGORY, dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', @@ -225,6 +231,7 @@ describe('getEntitiesAndGenerateAlerts', () => { [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: 'other', dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', @@ -241,18 +248,21 @@ describe('getEntitiesAndGenerateAlerts', () => { const currLocationMapWithThreeMore = new Map([...currLocationMap]).set('d', [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '789', dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '123', dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId2', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '456', dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId3', @@ -277,18 +287,21 @@ describe('getEntitiesAndGenerateAlerts', () => { const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: OTHER_CATEGORY, dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '123', dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '456', dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', @@ -310,18 +323,21 @@ describe('getEntitiesAndGenerateAlerts', () => { const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '123', dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: OTHER_CATEGORY, dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '456', dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', @@ -338,6 +354,7 @@ describe('getEntitiesAndGenerateAlerts', () => { new Map([...currLocationMap]).set('d', [ { location: [0, 0], + locationWkt: 'POINT (0 0)', shapeLocationId: '123', dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.ts index 4bf1aad7f536d6..c0d372e08dced3 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/get_entities_and_generate_alerts.ts @@ -56,6 +56,11 @@ export function getEntitiesAndGenerateAlerts( return; } + // TODO remove otherCatIndex check + // Elasticsearch filters aggregation is used to group results into buckets matching entity locations intersecting boundary shapes + // filters.other_bucket_key returns bucket with entities that did not intersect any boundary shape. + // shapeLocationId === OTHER_CATEGORY can only occur when containments.length === 1 + // test data does not follow this pattern and needs to be updated. const otherCatIndex = containments.findIndex( ({ shapeLocationId }) => shapeLocationId === OTHER_CATEGORY ); diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.test.ts index 6c87157ded42f9..91ec87fe196191 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.test.ts @@ -12,7 +12,7 @@ import { transformResults } from './transform_results'; describe('transformResults', () => { const dateField = '@timestamp'; const geoField = 'location'; - test('should correctly transform expected results', async () => { + test('should correctly transform expected results', () => { const transformedResults = transformResults( // @ts-ignore sampleAggsJsonResponse.body, @@ -27,13 +27,15 @@ describe('transformResults', () => { { dateInShape: '2021-04-28T16:56:11.923Z', docId: 'ZVBoGXkBsFLYN2Tj1wmV', - location: [-73.99018926545978, 40.751759740523994], + location: [0, 0], + locationWkt: 'POINT (-73.99018926545978 40.751759740523994)', shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk', }, { dateInShape: '2021-04-28T16:56:01.896Z', docId: 'YlBoGXkBsFLYN2TjsAlp', - location: [-73.98968475870788, 40.7506317878142], + location: [0, 0], + locationWkt: 'POINT (-73.98968475870788 40.7506317878142)', shapeLocationId: 'other', }, ], @@ -44,13 +46,15 @@ describe('transformResults', () => { { dateInShape: '2021-04-28T16:56:11.923Z', docId: 'ZlBoGXkBsFLYN2Tj1wmV', - location: [-73.99561604484916, 40.75449890457094], + location: [0, 0], + locationWkt: 'POINT (-73.99561604484916 40.75449890457094)', shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk', }, { dateInShape: '2021-04-28T16:56:01.896Z', docId: 'Y1BoGXkBsFLYN2TjsAlp', - location: [-73.99459345266223, 40.755913141183555], + location: [0, 0], + locationWkt: 'POINT (-73.99459345266223 40.755913141183555)', shapeLocationId: 'other', }, ], @@ -61,7 +65,8 @@ describe('transformResults', () => { { dateInShape: '2021-04-28T16:56:11.923Z', docId: 'Z1BoGXkBsFLYN2Tj1wmV', - location: [-73.98662586696446, 40.7667087810114], + location: [0, 0], + locationWkt: 'POINT (-73.98662586696446 40.7667087810114)', shapeLocationId: 'other', }, ], @@ -72,7 +77,7 @@ describe('transformResults', () => { const nestedDateField = 'time_data.@timestamp'; const nestedGeoField = 'geo.coords.location'; - test('should correctly transform expected results if fields are nested', async () => { + test('should correctly transform expected results if fields are nested', () => { const transformedResults = transformResults( // @ts-ignore sampleAggsJsonResponseWithNesting.body, @@ -87,7 +92,8 @@ describe('transformResults', () => { { dateInShape: '2020-09-28T18:01:41.190Z', docId: 'N-ng1XQB6yyY-xQxnGSM', - location: [-82.8814151789993, 40.62806099653244], + location: [0, 0], + locationWkt: 'POINT (-82.8814151789993 40.62806099653244)', shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', }, ], @@ -98,7 +104,8 @@ describe('transformResults', () => { { dateInShape: '2020-09-28T18:01:41.191Z', docId: 'iOng1XQB6yyY-xQxnGSM', - location: [-82.22068064846098, 39.006176185794175], + location: [0, 0], + locationWkt: 'POINT (-82.22068064846098 39.006176185794175)', shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', }, ], @@ -109,7 +116,8 @@ describe('transformResults', () => { { dateInShape: '2020-09-28T18:01:41.191Z', docId: 'n-ng1XQB6yyY-xQxnGSM', - location: [-84.71324851736426, 41.6677269525826], + location: [0, 0], + locationWkt: 'POINT (-84.71324851736426 41.6677269525826)', shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', }, ], @@ -120,7 +128,8 @@ describe('transformResults', () => { { dateInShape: '2020-09-28T18:01:41.192Z', docId: 'GOng1XQB6yyY-xQxnGWM', - location: [6.073727197945118, 39.07997465226799], + location: [0, 0], + locationWkt: 'POINT (6.073727197945118 39.07997465226799)', shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', }, ], diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.ts index d9b1c36fa48122..6b457575b40292 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/lib/transform_results.ts @@ -6,64 +6,55 @@ */ import _ from 'lodash'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { GeoContainmentAlertInstanceState } from '../types'; // Flatten agg results and get latest locations for each entity export function transformResults( - results: estypes.SearchResponse, + results: any, // eslint-disable-line @typescript-eslint/no-explicit-any dateField: string, geoField: string ): Map { - const buckets = _.get(results, 'aggregations.shapes.buckets', {}); - const arrResults = _.flatMap(buckets, (bucket: unknown, bucketKey: string) => { - const subBuckets = _.get(bucket, 'entitySplit.buckets', []); - return _.map(subBuckets, (subBucket) => { - const locationFieldResult = _.get( - subBucket, - `entityHits.hits.hits[0].fields["${geoField}"][0]`, - '' - ); - const location = locationFieldResult - ? _.chain(locationFieldResult) - .split(', ') - .map((coordString) => +coordString) - .reverse() - .value() - : []; - const dateInShape = _.get( - subBucket, - `entityHits.hits.hits[0].fields["${dateField}"][0]`, - null - ); - const docId = _.get(subBucket, `entityHits.hits.hits[0]._id`); + const resultsMap = new Map(); + const boundarySplitBuckets = results?.aggregations?.shapes?.buckets ?? {}; + for (const boundaryId in boundarySplitBuckets) { + if (!boundarySplitBuckets.hasOwnProperty(boundaryId)) { + continue; + } - return { - location, - shapeLocationId: bucketKey, - entityName: subBucket.key, - dateInShape, - docId, - }; - }); + const entitySplitBuckets = boundarySplitBuckets[boundaryId]?.entitySplit?.buckets ?? []; + for (let i = 0; i < entitySplitBuckets.length; i++) { + const entityName = entitySplitBuckets[i].key; + const entityResults = resultsMap.get(entityName) ?? []; + entityResults.push({ + // Required for zero down time (ZDT) + // populate legacy location so non-updated-kibana nodes can handle new alert state + // + // Why 0,0 vs parsing WKT and populating actual location? + // This loop gets processed for each entity location in each containing boundary, ie: its a hot loop + // There is a mimial amount of time between one kibana node updating and all Kibana nodes being updated + // vs a huge CPU penetatily for all kibana nodes for the rest of the time + // Algorithm optimized for the more common use case where all Kibana nodes are running updated version + location: [0, 0], + locationWkt: + entitySplitBuckets[i].entityHits?.hits?.hits?.[0]?.fields?.[geoField]?.[0] ?? '', + shapeLocationId: boundaryId, + dateInShape: + entitySplitBuckets[i].entityHits?.hits?.hits?.[0]?.fields?.[dateField]?.[0] ?? null, + docId: entitySplitBuckets[i].entityHits?.hits?.hits?.[0]?._id, + }); + resultsMap.set(entityName, entityResults); + } + } + + // TODO remove sort + // legacy algorithm sorted entity hits oldest to newest for an undocumented reason + // preserving sort to avoid unknown breaking changes + resultsMap.forEach((value, key) => { + if (value.length > 1) { + // sort oldest to newest + resultsMap.set(key, _.orderBy(value, ['dateInShape'], ['desc', 'asc'])); + } }); - const orderedResults = _.orderBy(arrResults, ['entityName', 'dateInShape'], ['asc', 'desc']) - // Get unique - .reduce( - ( - accu: Map, - el: GeoContainmentAlertInstanceState & { entityName: string } - ) => { - const { entityName, ...locationData } = el; - if (entityName) { - if (!accu.has(entityName)) { - accu.set(entityName, []); - } - accu.get(entityName)!.push(locationData); - } - return accu; - }, - new Map() - ); - return orderedResults; + + return resultsMap; } diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response.json b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response.json index b5751e527df4dd..401afa8d8807eb 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response.json +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response.json @@ -60,7 +60,7 @@ "2021-04-28T16:56:11.923Z" ], "location":[ - "40.751759740523994, -73.99018926545978" + "POINT (-73.99018926545978 40.751759740523994)" ], "entity_id":[ "0" @@ -94,7 +94,7 @@ "2021-04-28T16:56:11.923Z" ], "location":[ - "40.75449890457094, -73.99561604484916" + "POINT (-73.99561604484916 40.75449890457094)" ], "entity_id":[ "1" @@ -157,7 +157,7 @@ "2021-04-28T16:56:11.923Z" ], "location":[ - "40.7667087810114, -73.98662586696446" + "POINT (-73.98662586696446 40.7667087810114)" ], "entity_id":[ "2" @@ -191,7 +191,7 @@ "2021-04-28T16:56:01.896Z" ], "location":[ - "40.755913141183555, -73.99459345266223" + "POINT (-73.99459345266223 40.755913141183555)" ], "entity_id":[ "1" @@ -225,7 +225,7 @@ "2021-04-28T16:56:01.896Z" ], "location":[ - "40.7506317878142, -73.98968475870788" + "POINT (-73.98968475870788 40.7506317878142)" ], "entity_id":[ "0" diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response_with_nesting.json b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response_with_nesting.json index 9baf58465c38e1..b8973e185dabc5 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response_with_nesting.json +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/tests/es_sample_response_with_nesting.json @@ -46,7 +46,7 @@ "2020-09-28T18:01:41.190Z" ], "geo.coords.location" : [ - "40.62806099653244, -82.8814151789993" + "POINT (-82.8814151789993 40.62806099653244)" ], "entity_id" : [ "936" @@ -80,7 +80,7 @@ "2020-09-28T18:01:41.191Z" ], "geo.coords.location" : [ - "39.006176185794175, -82.22068064846098" + "POINT (-82.22068064846098 39.006176185794175)" ], "entity_id" : [ "AAL2019" @@ -114,7 +114,7 @@ "2020-09-28T18:01:41.191Z" ], "geo.coords.location" : [ - "41.6677269525826, -84.71324851736426" + "POINT (-84.71324851736426 41.6677269525826)" ], "entity_id" : [ "AAL2323" @@ -148,7 +148,7 @@ "2020-09-28T18:01:41.192Z" ], "geo.coords.location" : [ - "39.07997465226799, 6.073727197945118" + "POINT (6.073727197945118 39.07997465226799)" ], "entity_id" : [ "ABD5250" diff --git a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/types.ts b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/types.ts index 236688d504efe9..00d09051d1007d 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/types.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/geo_containment/types.ts @@ -54,7 +54,11 @@ export interface GeoContainmentRuleState extends RuleTypeState { } export interface GeoContainmentAlertInstanceState extends AlertInstanceState { + // 8.10-, location is [lon, lat] array. + // continue to populate 'location' for backwards compatibility with persisted alert instance state and ZDT location: number[]; + // 8.11+, location will be wkt represenation of geometry. + locationWkt?: string; shapeLocationId: string; dateInShape: string | null; docId: string;