From bdac93cdb19a30e2d59a7ec363cc4fedce7e357d Mon Sep 17 00:00:00 2001 From: nnamdifrankie Date: Wed, 29 Jan 2020 13:11:29 -0500 Subject: [PATCH 1/5] EMT-67: add kql support for endpoint list --- .../endpoint/server/routes/endpoints.test.ts | 65 ++++++++++++++++++- .../endpoint/server/routes/endpoints.ts | 28 +++++--- .../endpoint/endpoint_query_builders.test.ts | 59 +++++++++++++++++ .../endpoint/endpoint_query_builders.ts | 17 ++++- .../apis/endpoint/endpoints.ts | 36 +++++++++- 5 files changed, 191 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts index 04a38972401ed9..68a22353cfe4eb 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts @@ -79,7 +79,7 @@ describe('test endpoint route', () => { expect(endpointResultList.request_page_size).toEqual(10); }); - it('test find the latest of all endpoints with params', async () => { + it('test find the latest of all endpoints with paging properties', async () => { const mockRequest = httpServerMock.createKibanaRequest({ body: { paging_properties: [ @@ -112,6 +112,69 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({ + match_all: {}, + }); + expect(routeConfig.options).toEqual({ authRequired: true }); + expect(mockResponse.ok).toBeCalled(); + const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList; + expect(endpointResultList.endpoints.length).toEqual(2); + expect(endpointResultList.total).toEqual(2); + expect(endpointResultList.request_page_index).toEqual(10); + expect(endpointResultList.request_page_size).toEqual(10); + }); + + it('test find the latest of all endpoints with paging and filters properties', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + body: { + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 1, + }, + ], + + filters: 'not host.ip:10.140.73.246', + }, + }); + mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => + Promise.resolve((data as unknown) as SearchResponse) + ); + [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/endpoints') + )!; + + await routeHandler( + ({ + core: { + elasticsearch: { + dataClient: mockScopedClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + + expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({ + bool: { + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.ip': '10.140.73.246', + }, + }, + ], + }, + }, + }, + }); expect(routeConfig.options).toEqual({ authRequired: true }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList; diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.ts b/x-pack/plugins/endpoint/server/routes/endpoints.ts index 4fc3e653f94265..5f056169c4d423 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.ts @@ -23,16 +23,26 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp validate: { body: schema.nullable( schema.object({ - paging_properties: schema.arrayOf( - schema.oneOf([ - // the number of results to return for this request per page - schema.object({ - page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }), - }), - // the index of the page to return - schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }), - ]) + paging_properties: schema.nullable( + schema.arrayOf( + schema.oneOf([ + /** + * the number of results to return for this request per page + */ + schema.object({ + page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }), + }), + /** + * the index of the first result in the page in the total matching set + */ + schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }), + ]) + ) ), + /** + * filter to be applied, it could be a kql expression or discrete filter to be implemented + */ + filters: schema.nullable(schema.oneOf([schema.string()])), }) ), }, diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts index 3c931a251d6977..57d7ed27e98fe6 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts @@ -51,4 +51,63 @@ describe('test query builder', () => { } as Record); }); }); + + describe('test query builder with kql filter', () => { + it('test default query params for all endpoints when no params or body is provided', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + body: { + filters: 'not host.ip:10.140.73.246', + }, + }); + const query = await kibanaRequestToEndpointListQuery(mockRequest, { + logFactory: loggingServiceMock.create(), + config: () => Promise.resolve(EndpointConfigSchema.validate({})), + }); + expect(query).toEqual({ + body: { + query: { + bool: { + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.ip': '10.140.73.246', + }, + }, + ], + }, + }, + }, + }, + collapse: { + field: 'host.id.keyword', + inner_hits: { + name: 'most_recent', + size: 1, + sort: [{ 'event.created': 'desc' }], + }, + }, + aggs: { + total: { + cardinality: { + field: 'host.id.keyword', + }, + }, + }, + sort: [ + { + 'event.created': { + order: 'desc', + }, + }, + ], + }, + from: 0, + size: 10, + index: 'endpoint-agent*', + } as Record); + }); + }); }); diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts index 102c268cf9ec4c..6c487c88fcb039 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts @@ -6,6 +6,10 @@ import { KibanaRequest } from 'kibana/server'; import { EndpointAppConstants } from '../../../common/types'; import { EndpointAppContext } from '../../types'; +import { + fromKueryExpression, + toElasticsearchQuery, +} from '../../../../../../src/plugins/data/common/es_query/kuery/ast'; export const kibanaRequestToEndpointListQuery = async ( request: KibanaRequest, @@ -14,9 +18,7 @@ export const kibanaRequestToEndpointListQuery = async ( const pagingProperties = await getPagingProperties(request, endpointAppContext); return { body: { - query: { - match_all: {}, - }, + query: buildQueryBody(request), collapse: { field: 'host.id.keyword', inner_hits: { @@ -65,3 +67,12 @@ async function getPagingProperties( pageIndex: pagingProperties.page_index || config.endpointResultListDefaultFirstPageIndex, }; } + +function buildQueryBody(request: KibanaRequest): Record { + if (request?.body?.filters && typeof request.body.filters === 'string') { + return toElasticsearchQuery(fromKueryExpression(request.body.filters)); + } + return { + match_all: {}, + }; +} diff --git a/x-pack/test/api_integration/apis/endpoint/endpoints.ts b/x-pack/test/api_integration/apis/endpoint/endpoints.ts index 1c520fe92e38ee..e367a1b2cd193d 100644 --- a/x-pack/test/api_integration/apis/endpoint/endpoints.ts +++ b/x-pack/test/api_integration/apis/endpoint/endpoints.ts @@ -40,7 +40,7 @@ export default function({ getService }: FtrProviderContext) { expect(body.request_page_index).to.eql(0); }); - it('endpoints api should return page based on params passed.', async () => { + it('endpoints api should return page based on paging properties passed.', async () => { const { body } = await supertest .post('/api/endpoint/endpoints') .set('kbn-xsrf', 'xxx') @@ -102,6 +102,40 @@ export default function({ getService }: FtrProviderContext) { .expect(400); expect(body.message).to.contain('Value is [0] but it must be equal to or greater than [1]'); }); + + it('endpoints api should return page based on filters passed.', async () => { + const { body } = await supertest + .post('/api/endpoint/endpoints') + .set('kbn-xsrf', 'xxx') + .send({ filters: 'not host.ip:10.101.149.26' }) + .expect(200); + expect(body.total).to.eql(2); + expect(body.endpoints.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); + + it('endpoints api should return page based on filters and paging passed.', async () => { + const { body } = await supertest + .post('/api/endpoint/endpoints') + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 1, + }, + { + page_index: 1, + }, + ], + filters: 'not host.ip:10.101.149.26', + }) + .expect(200); + expect(body.total).to.eql(2); + expect(body.endpoints.length).to.eql(1); + expect(body.request_page_size).to.eql(1); + expect(body.request_page_index).to.eql(1); + }); }); }); } From ac0a0f79ecfeab9ab2e84a0312bbfaf03066a813 Mon Sep 17 00:00:00 2001 From: nnamdifrankie Date: Sun, 2 Feb 2020 20:17:18 -0500 Subject: [PATCH 2/5] EMt-67: clean up from PR review --- x-pack/plugins/endpoint/server/routes/endpoints.test.ts | 2 +- x-pack/plugins/endpoint/server/routes/endpoints.ts | 4 ++-- .../server/services/endpoint/endpoint_query_builders.test.ts | 2 +- .../server/services/endpoint/endpoint_query_builders.ts | 4 ++-- x-pack/test/api_integration/apis/endpoint/endpoints.ts | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts index 0966e1f321f08e..25c4225495a41a 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts @@ -136,7 +136,7 @@ describe('test endpoint route', () => { }, ], - filters: 'not host.ip:10.140.73.246', + filter: 'not host.ip:10.140.73.246', }, }); mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.ts b/x-pack/plugins/endpoint/server/routes/endpoints.ts index 48edbc22a3e368..054172a7f258aa 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.ts @@ -36,7 +36,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }), }), /** - * the index of the first result in the page in the total matching set + * the zero based page index of the the total number of pages of page size */ schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }), ]) @@ -45,7 +45,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp /** * filter to be applied, it could be a kql expression or discrete filter to be implemented */ - filters: schema.nullable(schema.oneOf([schema.string()])), + filter: schema.nullable(schema.oneOf([schema.string()])), }) ), }, diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts index 80af11f4f6f251..bd9986ecf1f976 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts @@ -59,7 +59,7 @@ describe('query builder', () => { it('test default query params for all endpoints when no params or body is provided', async () => { const mockRequest = httpServerMock.createKibanaRequest({ body: { - filters: 'not host.ip:10.140.73.246', + filter: 'not host.ip:10.140.73.246', }, }); const query = await kibanaRequestToEndpointListQuery(mockRequest, { diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts index bba53e18746fae..e0e51ddb810084 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts @@ -69,8 +69,8 @@ async function getPagingProperties( } function buildQueryBody(request: KibanaRequest): Record { - if (request?.body?.filters && typeof request.body.filters === 'string') { - return toElasticsearchQuery(fromKueryExpression(request.body.filters)); + if (typeof request?.body?.filter === 'string') { + return toElasticsearchQuery(fromKueryExpression(request.body.filter)); } return { match_all: {}, diff --git a/x-pack/test/api_integration/apis/endpoint/endpoints.ts b/x-pack/test/api_integration/apis/endpoint/endpoints.ts index e367a1b2cd193d..c0618163d7b276 100644 --- a/x-pack/test/api_integration/apis/endpoint/endpoints.ts +++ b/x-pack/test/api_integration/apis/endpoint/endpoints.ts @@ -107,7 +107,7 @@ export default function({ getService }: FtrProviderContext) { const { body } = await supertest .post('/api/endpoint/endpoints') .set('kbn-xsrf', 'xxx') - .send({ filters: 'not host.ip:10.101.149.26' }) + .send({ filter: 'not host.ip:10.101.149.26' }) .expect(200); expect(body.total).to.eql(2); expect(body.endpoints.length).to.eql(2); @@ -128,7 +128,7 @@ export default function({ getService }: FtrProviderContext) { page_index: 1, }, ], - filters: 'not host.ip:10.101.149.26', + filter: 'not host.ip:10.101.149.26', }) .expect(200); expect(body.total).to.eql(2); From 93cc54d3015ed5ffa92fbfbf25d0f60c7a2bebc5 Mon Sep 17 00:00:00 2001 From: nnamdifrankie Date: Mon, 3 Feb 2020 13:06:43 -0500 Subject: [PATCH 3/5] EMT-67: improve testing --- .../api_integration/apis/endpoint/endpoints.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/endpoints.ts b/x-pack/test/api_integration/apis/endpoint/endpoints.ts index c0618163d7b276..1b0f63d85e5233 100644 --- a/x-pack/test/api_integration/apis/endpoint/endpoints.ts +++ b/x-pack/test/api_integration/apis/endpoint/endpoints.ts @@ -116,25 +116,29 @@ export default function({ getService }: FtrProviderContext) { }); it('endpoints api should return page based on filters and paging passed.', async () => { + const notIncludedIp = '10.101.149.26'; const { body } = await supertest .post('/api/endpoint/endpoints') .set('kbn-xsrf', 'xxx') .send({ paging_properties: [ { - page_size: 1, + page_size: 10, }, { - page_index: 1, + page_index: 0, }, ], - filter: 'not host.ip:10.101.149.26', + filter: `not host.ip:${notIncludedIp}`, }) .expect(200); expect(body.total).to.eql(2); - expect(body.endpoints.length).to.eql(1); - expect(body.request_page_size).to.eql(1); - expect(body.request_page_index).to.eql(1); + const availableIps = [].concat(...body.endpoints.map(metadata => metadata.host.ip)); + expect(availableIps).to.eql(['10.192.213.130', '10.70.28.129', '10.46.229.234']); + expect(availableIps).not.include.eql(notIncludedIp); + expect(body.endpoints.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); }); }); }); From 6d436b6b0fdc98b8910d11522c76e2950e877fc4 Mon Sep 17 00:00:00 2001 From: nnamdifrankie Date: Mon, 3 Feb 2020 14:58:21 -0500 Subject: [PATCH 4/5] EMT-67: fix test type --- x-pack/test/api_integration/apis/endpoint/endpoints.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/endpoints.ts b/x-pack/test/api_integration/apis/endpoint/endpoints.ts index 1b0f63d85e5233..210e9d78d7e181 100644 --- a/x-pack/test/api_integration/apis/endpoint/endpoints.ts +++ b/x-pack/test/api_integration/apis/endpoint/endpoints.ts @@ -133,9 +133,11 @@ export default function({ getService }: FtrProviderContext) { }) .expect(200); expect(body.total).to.eql(2); - const availableIps = [].concat(...body.endpoints.map(metadata => metadata.host.ip)); - expect(availableIps).to.eql(['10.192.213.130', '10.70.28.129', '10.46.229.234']); - expect(availableIps).not.include.eql(notIncludedIp); + const resultIps: string[] = [].concat( + ...body.endpoints.map((metadata: Record) => metadata.host.ip) + ); + expect(resultIps).to.eql(['10.192.213.130', '10.70.28.129', '10.46.229.234']); + expect(resultIps).not.include.eql(notIncludedIp); expect(body.endpoints.length).to.eql(2); expect(body.request_page_size).to.eql(10); expect(body.request_page_index).to.eql(0); From a8a8c85f3bdc9b6d7e40428a7c9d108b4656b929 Mon Sep 17 00:00:00 2001 From: nnamdifrankie Date: Tue, 4 Feb 2020 12:34:25 -0500 Subject: [PATCH 5/5] EMT-67: change import for external dependency --- .../server/services/endpoint/endpoint_query_builders.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts index e0e51ddb810084..c143b09ec453c3 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts @@ -6,10 +6,7 @@ import { KibanaRequest } from 'kibana/server'; import { EndpointAppConstants } from '../../../common/types'; import { EndpointAppContext } from '../../types'; -import { - fromKueryExpression, - toElasticsearchQuery, -} from '../../../../../../src/plugins/data/common/es_query/kuery/ast'; +import { esKuery } from '../../../../../../src/plugins/data/server'; export const kibanaRequestToEndpointListQuery = async ( request: KibanaRequest, @@ -70,7 +67,7 @@ async function getPagingProperties( function buildQueryBody(request: KibanaRequest): Record { if (typeof request?.body?.filter === 'string') { - return toElasticsearchQuery(fromKueryExpression(request.body.filter)); + return esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(request.body.filter)); } return { match_all: {},