From 4e0c11ea406c1b100b403a01a9f24d63ee31b725 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Tue, 28 Apr 2020 12:37:25 -0400 Subject: [PATCH] [Event Log] use @timestamp field for queries (#64391) resolves https://github.com/elastic/kibana/issues/64275 Changes the fields used to query the event log by time range to use the `@timestamp` field. Also allow `@timestamp` as a sort option, and make it the default sort option. --- .../server/es/cluster_client_adapter.test.ts | 8 +-- .../server/es/cluster_client_adapter.ts | 4 +- .../event_log/server/event_log_client.test.ts | 4 +- .../event_log/server/event_log_client.ts | 3 +- .../event_log/public_api_integration.ts | 51 ++++++------------- .../event_log/service_api_integration.ts | 3 +- 6 files changed, 28 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 470123ada48ea4..f79962a3241317 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -226,7 +226,7 @@ describe('queryEventsBySavedObject', () => { body: { from: 0, size: 10, - sort: { 'event.start': { order: 'asc' } }, + sort: { '@timestamp': { order: 'asc' } }, query: { bool: { must: [ @@ -340,7 +340,7 @@ describe('queryEventsBySavedObject', () => { }, { range: { - 'event.start': { + '@timestamp': { gte: start, }, }, @@ -409,14 +409,14 @@ describe('queryEventsBySavedObject', () => { }, { range: { - 'event.start': { + '@timestamp': { gte: start, }, }, }, { range: { - 'event.end': { + '@timestamp': { lte: end, }, }, diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 6d5c6b31a637c3..47d273b9981e32 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -176,14 +176,14 @@ export class ClusterClientAdapter { }, start && { range: { - 'event.start': { + '@timestamp': { gte: start, }, }, }, end && { range: { - 'event.end': { + '@timestamp': { lte: end, }, }, diff --git a/x-pack/plugins/event_log/server/event_log_client.test.ts b/x-pack/plugins/event_log/server/event_log_client.test.ts index 6d4c9b67abc1ba..17c073c4b27f9b 100644 --- a/x-pack/plugins/event_log/server/event_log_client.test.ts +++ b/x-pack/plugins/event_log/server/event_log_client.test.ts @@ -112,7 +112,7 @@ describe('EventLogStart', () => { { page: 1, per_page: 10, - sort_field: 'event.start', + sort_field: '@timestamp', sort_order: 'asc', } ); @@ -193,7 +193,7 @@ describe('EventLogStart', () => { { page: 1, per_page: 10, - sort_field: 'event.start', + sort_field: '@timestamp', sort_order: 'asc', start, end, diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 765f0895f8e0d1..8ef245e60aadc4 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -36,6 +36,7 @@ export const findOptionsSchema = schema.object({ end: optionalDateFieldSchema, sort_field: schema.oneOf( [ + schema.literal('@timestamp'), schema.literal('event.start'), schema.literal('event.end'), schema.literal('event.provider'), @@ -44,7 +45,7 @@ export const findOptionsSchema = schema.object({ schema.literal('message'), ], { - defaultValue: 'event.start', + defaultValue: '@timestamp', } ), sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts index d664357c3ba126..e5b840b335846f 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts @@ -7,7 +7,7 @@ import { merge, omit, times, chunk, isEmpty } from 'lodash'; import uuid from 'uuid'; import expect from '@kbn/expect/expect.js'; -import moment, { Moment } from 'moment'; +import moment from 'moment'; import { FtrProviderContext } from '../../ftr_provider_context'; import { IEvent } from '../../../../plugins/event_log/server'; import { IValidatedEvent } from '../../../../plugins/event_log/server/types'; @@ -43,10 +43,8 @@ export default function({ getService }: FtrProviderContext) { it('should support pagination for events', async () => { const id = uuid.v4(); - const timestamp = moment(); - const [firstExpectedEvent, ...expectedEvents] = times(6, () => - fakeEvent(id, fakeEventTiming(timestamp.add(1, 's'))) - ); + const [firstExpectedEvent, ...expectedEvents] = times(6, () => fakeEvent(id)); + // run one first to create the SO and avoid clashes await logTestEvent(id, firstExpectedEvent); await Promise.all(expectedEvents.map(event => logTestEvent(id, event))); @@ -82,10 +80,7 @@ export default function({ getService }: FtrProviderContext) { it('should support sorting by event end', async () => { const id = uuid.v4(); - const timestamp = moment(); - const [firstExpectedEvent, ...expectedEvents] = times(6, () => - fakeEvent(id, fakeEventTiming(timestamp.add(1, 's'))) - ); + const [firstExpectedEvent, ...expectedEvents] = times(6, () => fakeEvent(id)); // run one first to create the SO and avoid clashes await logTestEvent(id, firstExpectedEvent); await Promise.all(expectedEvents.map(event => logTestEvent(id, event))); @@ -106,21 +101,24 @@ export default function({ getService }: FtrProviderContext) { it('should support date ranges for events', async () => { const id = uuid.v4(); - const timestamp = moment(); - - const firstEvent = fakeEvent(id, fakeEventTiming(timestamp)); + // write a document that shouldn't be found in the inclusive date range search + const firstEvent = fakeEvent(id); await logTestEvent(id, firstEvent); - await delay(100); - const start = timestamp.add(1, 's').toISOString(); + // wait a second, get the start time for the date range search + await delay(1000); + const start = new Date().toISOString(); - const expectedEvents = times(6, () => fakeEvent(id, fakeEventTiming(timestamp.add(1, 's')))); + // write the documents that we should be found in the date range searches + const expectedEvents = times(6, () => fakeEvent(id)); await Promise.all(expectedEvents.map(event => logTestEvent(id, event))); - const end = timestamp.add(1, 's').toISOString(); + // get the end time for the date range search + const end = new Date().toISOString(); - await delay(100); - const lastEvent = fakeEvent(id, fakeEventTiming(timestamp.add(1, 's'))); + // write a document that shouldn't be found in the inclusive date range search + await delay(1000); + const lastEvent = fakeEvent(id); await logTestEvent(id, lastEvent); await retry.try(async () => { @@ -195,29 +193,12 @@ export default function({ getService }: FtrProviderContext) { .expect(200); } - function fakeEventTiming(start: Moment): Partial { - return { - event: { - start: start.toISOString(), - end: start - .clone() - .add(500, 'milliseconds') - .toISOString(), - }, - }; - } - function fakeEvent(id: string, overrides: Partial = {}): IEvent { - const start = moment().toISOString(); - const end = moment().toISOString(); return merge( { event: { provider: 'event_log_fixture', action: 'test', - start, - end, - duration: 1000000, }, kibana: { saved_objects: [ diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts index 2de395308ce74c..31668e8345275e 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import uuid from 'uuid'; import expect from '@kbn/expect/expect.js'; import { IEvent } from '../../../../plugins/event_log/server'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -97,7 +98,7 @@ export default function({ getService }: FtrProviderContext) { await registerProviderActions('provider4', ['action1', 'action2']); } - const eventId = '1'; + const eventId = uuid.v4(); const event: IEvent = { event: { action: 'action1', provider: 'provider4' }, kibana: { saved_objects: [{ type: 'event_log_test', id: eventId }] },