From 1e39db3adfaa4caf76e1c9e8b4a372b1e1d01f4b Mon Sep 17 00:00:00 2001 From: sulemanof Date: Thu, 10 Jan 2019 16:22:42 +0300 Subject: [PATCH 01/58] Added a feature of rollup search on the UI side Signed-off-by: Alexey Antonov --- .../public/components/index_pattern.js | 20 ++++++++++++++++++- .../metrics/public/kbn_vis_types/index.js | 3 ++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/legacy/core_plugins/metrics/public/components/index_pattern.js b/src/legacy/core_plugins/metrics/public/components/index_pattern.js index dc456026acb5ac..8c9188bea74d5d 100644 --- a/src/legacy/core_plugins/metrics/public/components/index_pattern.js +++ b/src/legacy/core_plugins/metrics/public/components/index_pattern.js @@ -42,11 +42,13 @@ export const IndexPattern = props => { const indexPatternName = `${prefix}index_pattern`; const intervalName = `${prefix}interval`; const dropBucketName = `${prefix}drop_last_bucket`; + const rollupSearchName = `${prefix}rollup_search`; const defaults = { [indexPatternName]: '*', [intervalName]: 'auto', - [dropBucketName]: 1 + [dropBucketName]: 1, + [rollupSearchName]: 0 }; const htmlId = htmlIdGenerator(); @@ -73,6 +75,22 @@ export const IndexPattern = props => { /> + + + + + + + + + Date: Mon, 14 Jan 2019 12:36:53 +0300 Subject: [PATCH 02/58] Rollup Feature - initial commit --- src/legacy/core_plugins/metrics/index.js | 11 +-- .../public/components/index_pattern.js | 5 ++ .../server/lib/search_strategies/index.js | 24 +++++++ .../search_strategies_register.js | 45 ++++++++++++ .../searh_requests/abstract_request.js | 28 ++++++++ .../searh_requests/multi_search_request.js | 34 +++++++++ .../strategies/abstract_search_strategy.js | 27 +++++++ .../strategies/default_search_strategy.js | 44 ++++++++++++ .../server/lib/vis_data/get_series_data.js | 70 +++++++++---------- .../lib/vis_data/series/get_request_params.js | 27 ++++--- x-pack/plugins/rollup/index.js | 11 ++- .../server/lib/search_strategies/index.js | 8 +++ .../register_rollup_search_strategy.js | 16 +++++ .../rollup_search_request.js | 26 +++++++ .../rollup_search_strategy.js | 25 +++++++ 15 files changed, 349 insertions(+), 52 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/index.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js diff --git a/src/legacy/core_plugins/metrics/index.js b/src/legacy/core_plugins/metrics/index.js index 1258cedf8c7f15..c71cbb6dcfeceb 100644 --- a/src/legacy/core_plugins/metrics/index.js +++ b/src/legacy/core_plugins/metrics/index.js @@ -21,14 +21,16 @@ import { resolve } from 'path'; import fieldsRoutes from './server/routes/fields'; import visDataRoutes from './server/routes/vis'; +import SearchStrategiesRegister from './server/lib/search_strategies/search_strategies_register'; export default function (kibana) { return new kibana.Plugin({ + id: 'metrics', require: ['kibana', 'elasticsearch'], uiExports: { visTypes: [ - 'plugins/metrics/kbn_vis_types' + 'plugins/metrics/kbn_vis_types', ], styleSheetPaths: resolve(__dirname, 'public/index.scss'), }, @@ -37,16 +39,15 @@ export default function (kibana) { return Joi.object({ enabled: Joi.boolean().default(true), chartResolution: Joi.number().default(150), - minimumBucketSize: Joi.number().default(10) + minimumBucketSize: Joi.number().default(10), }).default(); }, - init(server) { fieldsRoutes(server); visDataRoutes(server); - } - + SearchStrategiesRegister.init(server); + }, }); } diff --git a/src/legacy/core_plugins/metrics/public/components/index_pattern.js b/src/legacy/core_plugins/metrics/public/components/index_pattern.js index 8c9188bea74d5d..70b21f92ff4f64 100644 --- a/src/legacy/core_plugins/metrics/public/components/index_pattern.js +++ b/src/legacy/core_plugins/metrics/public/components/index_pattern.js @@ -75,6 +75,11 @@ export const IndexPattern = props => { /> + + {/* Question: I thinks it's better to return from the server the list + of available Search Strategies in accordance of current licence. In case + of 2 or more values user can select which strategy we should use. + */} searchStrategiesRegister.add(searchStrategy)); + + return searchStrategiesRegister.add(new DefaultSearchStrategy(server)); + } + + static getStrategyForIndex(indexPattern) { + return strategies.find(searchStrategy => { + return searchStrategy.isViable(indexPattern); + }); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js new file mode 100644 index 00000000000000..0f33be24955da2 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export default class AbstractSearchRequest { + constructor(req, callWithRequest) { + this.req = req; + this.callWithRequest = callWithRequest; + } + + search() { + throw new Error('AbstractSearchRequest: search method should be defined'); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js new file mode 100644 index 00000000000000..663935b9371f22 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchRequest from './abstract_request'; + +const SEARCH_METHOD = 'msearch'; + +export default class MultiSearchRequest extends AbstractSearchRequest { + async search(options) { + const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); + const { responses } = await this.callWithRequest(this.req, SEARCH_METHOD, { + ...options, + rest_total_hits_as_int: true, + ignore_throttled: !includeFrozen, + }); + + return responses; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js new file mode 100644 index 00000000000000..b44a04880ca0b3 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export default class AbstractSearchStrategy { + constructor(server, callWithRequestFactory, SearchRequest) { + this.getSearchRequest = (req) => { + const callWithRequest = callWithRequestFactory(server, req); + + return new SearchRequest(req, callWithRequest); + }; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js new file mode 100644 index 00000000000000..499a879660bfd5 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchStrategy from './abstract_search_strategy'; +import MultiSearchRequest from '../searh_requests/multi_search_request'; +import { i18n } from '@kbn/i18n'; + +const callWithRequestFactory = (server, request) => { + const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); + + return callWithRequest; +}; + +export default class DefaultSearchStrategy extends AbstractSearchStrategy { + name = 'default'; + batchRequestsSupport = true; + + label = i18n.translate('tsvb.searchStrategies.default.label', { + defaultMessage: 'Default', + }); + + constructor(server) { + super(server, callWithRequestFactory, MultiSearchRequest); + } + + isViable(indexPattern) { + return Boolean(indexPattern); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 2594db2c5db6d2..faf837db40a4f6 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -16,50 +16,50 @@ * specific language governing permissions and limitations * under the License. */ - import getRequestParams from './series/get_request_params'; import handleResponseBody from './series/handle_response_body'; import handleErrorResponse from './handle_error_response'; import getAnnotations from './get_annotations'; +import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; + export async function getSeriesData(req, panel) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); + const indexPattern = panel.index_pattern; + const searchStrategy = SearchStrategiesRegister.getStrategyForIndex(indexPattern); + const searchRequest = searchStrategy.getSearchRequest(req); const esQueryConfig = await getEsQueryConfig(req); - try { - const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig)); - const bodies = await Promise.all(bodiesPromises); - const params = { - rest_total_hits_as_int: true, - ignore_throttled: !includeFrozen, - body: bodies.reduce((acc, items) => acc.concat(items), []) - }; - return callWithRequest(req, 'msearch', params) - .then(resp => { - const series = resp.responses.map(handleResponseBody(panel)); - return { - [panel.id]: { - id: panel.id, - series: series.reduce((acc, series) => acc.concat(series), []) - } - }; - }) - .then(resp => { - if (!panel.annotations || panel.annotations.length === 0) return resp; - return getAnnotations(req, panel, esQueryConfig).then(annotations => { - resp[panel.id].annotations = annotations; - return resp; - }); - }) - .then(resp => { - resp.type = panel.type; + const body = panel.series + .map(series => getRequestParams(req, panel, series, searchStrategy.batchRequestsSupport)) + .reduce((acc, items) => acc.concat(items), []); + + // todo: + // try { + return searchRequest.search({ body }, indexPattern) + .then(data => { + const series = data.map(handleResponseBody(panel, esQueryConfig)); + return { + [panel.id]: { + id: panel.id, + series: series.reduce((acc, series) => acc.concat(series), []), + }, + }; + }) + .then(resp => { + // TODO: don't forget to remove + // Should be refactored + return resp; + if (!panel.annotations || panel.annotations.length === 0) return resp; + return getAnnotations(req, panel, esQueryConfig).then(annotations => { + resp[panel.id].annotations = annotations; return resp; - }) - .catch(handleErrorResponse(panel)); - } catch(e) { - return handleErrorResponse(e); - } + }); + }) + .then(resp => { + resp.type = panel.type; + return resp; + }) + .catch(handleErrorResponse(panel)); } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index 598ebed73b94cd..23106313e6d728 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -16,20 +16,25 @@ * specific language governing permissions and limitations * under the License. */ - import buildRequestBody from './build_request_body'; -import { getIndexPatternObject } from '../helpers/get_index_pattern'; -export default async (req, panel, series, esQueryConfig) => { - const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; +export default (req, panel, series, isBatchRequest = true) => { + const bodies = []; const indexPatternObject = await getIndexPatternObject(req, indexPatternString); - const request = buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject); - request.timeout = '90s'; - return [ - { + + if (isBatchRequest) { + const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; + + bodies.push({ index: indexPatternString, ignoreUnavailable: true, - }, - request, - ]; + }); + } + + bodies.push({ + ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject) + timeout: '90s' + }); + + return bodies; }; diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index 6e15bab5f48c2b..1fcb732beeed43 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -8,6 +8,7 @@ import { resolve } from 'path'; import { PLUGIN } from './common'; import { registerLicenseChecker } from './server/lib/register_license_checker'; import { rollupDataEnricher } from './rollup_data_enricher'; +import registerRollupSearchStrategy from './server/lib/search_strategies'; import { registerIndicesRoute, registerFieldsForWildcardRoute, @@ -20,7 +21,14 @@ export function rollup(kibana) { return new kibana.Plugin({ id: PLUGIN.ID, publicDir: resolve(__dirname, 'public'), - require: ['kibana', 'elasticsearch', 'xpack_main'], + + // !!!!!!! + // Question: not sure that it's a good idea to have a reference to 'metrics' from here + // but if we don't add this one we cannot guarantee that 'metrics' plugin was initialized + // + // Maybe we should move 'search strategies' from the 'metrics' folder. + // !!!!!!! + require: ['kibana', 'elasticsearch', 'xpack_main', 'metrics'], uiExports: { styleSheetPaths: resolve(__dirname, 'public/index.scss'), managementSections: [ @@ -60,6 +68,7 @@ export function rollup(kibana) { ) { server.plugins.index_management.addIndexManagementDataEnricher(rollupDataEnricher); } + registerRollupSearchStrategy(server); } }); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/index.js b/x-pack/plugins/rollup/server/lib/search_strategies/index.js new file mode 100644 index 00000000000000..824d9fce2395f3 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/index.js @@ -0,0 +1,8 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import registerRollupSearchStrategy from './register_rollup_search_strategy'; + +export default registerRollupSearchStrategy; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js new file mode 100644 index 00000000000000..cce2cb292e8310 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -0,0 +1,16 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import RollupSearchStrategy from './rollup_search_strategy'; + +export default (server) => { + const { addSearchStrategy } = server.plugins.metrics; + + if (addSearchStrategy) { + addSearchStrategy(new RollupSearchStrategy(server)); + } + + return server; +}; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js new file mode 100644 index 00000000000000..cfafc393097638 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js @@ -0,0 +1,26 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ + +// !!!!!!! +// Question: is it ok? how to make url a little bit more friendly for developers. Can I create alias? +// !!!!!!! +import { AbstractSearchRequest } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; + +const SEARCH_METHOD = 'rollup.search'; + +export default class RollupSearchRequest extends AbstractSearchRequest { + async search(options, index) { + const bodies = Array.isArray(options.body) ? options.body : [options.body]; + const requests = bodies + .map(body => this.callWithRequest(SEARCH_METHOD, { + body, + index, + rest_total_hits_as_int: true, + })); + + return await Promise.all(requests); + } +} diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js new file mode 100644 index 00000000000000..ebc80a5b857eee --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -0,0 +1,25 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { i18n } from '@kbn/i18n'; +import RollupSearchRequest from './rollup_search_request'; +import { callWithRequestFactory } from '../call_with_request_factory'; +import { AbstractSearchStrategy } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; + +export default class RollupSearchStrategy extends AbstractSearchStrategy { + name = 'rollup'; + batchRequestsSupport = false; + label = i18n.translate('tsvb.searchStrategies.rollup.label', { + defaultMessage: 'Rollup', + }); + + constructor(server) { + super(server, callWithRequestFactory, RollupSearchRequest); + } + + isViable(indexPattern) { + return Boolean(indexPattern); + } +} From f07357f16cb839279228ceb515341984cd815f03 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Tue, 15 Jan 2019 11:03:13 +0300 Subject: [PATCH 03/58] Revert "Added a feature of rollup search on the UI side" This reverts commit 9568b0970b16f5102f50b748bb4d691a8612c2c2. # Conflicts: # src/legacy/core_plugins/metrics/public/components/index_pattern.js --- .../public/components/index_pattern.js | 25 +------------------ .../metrics/public/kbn_vis_types/index.js | 3 +-- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/legacy/core_plugins/metrics/public/components/index_pattern.js b/src/legacy/core_plugins/metrics/public/components/index_pattern.js index 70b21f92ff4f64..dc456026acb5ac 100644 --- a/src/legacy/core_plugins/metrics/public/components/index_pattern.js +++ b/src/legacy/core_plugins/metrics/public/components/index_pattern.js @@ -42,13 +42,11 @@ export const IndexPattern = props => { const indexPatternName = `${prefix}index_pattern`; const intervalName = `${prefix}interval`; const dropBucketName = `${prefix}drop_last_bucket`; - const rollupSearchName = `${prefix}rollup_search`; const defaults = { [indexPatternName]: '*', [intervalName]: 'auto', - [dropBucketName]: 1, - [rollupSearchName]: 0 + [dropBucketName]: 1 }; const htmlId = htmlIdGenerator(); @@ -75,27 +73,6 @@ export const IndexPattern = props => { /> - - {/* Question: I thinks it's better to return from the server the list - of available Search Strategies in accordance of current licence. In case - of 2 or more values user can select which strategy we should use. - */} - - - - - - - - - Date: Tue, 15 Jan 2019 11:09:21 +0300 Subject: [PATCH 04/58] Remove the 'label' property from the search strategies --- .../search_strategies/strategies/default_search_strategy.js | 5 ----- .../server/lib/search_strategies/rollup_search_strategy.js | 4 ---- 2 files changed, 9 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index 499a879660bfd5..554f8438cc0568 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -18,7 +18,6 @@ */ import AbstractSearchStrategy from './abstract_search_strategy'; import MultiSearchRequest from '../searh_requests/multi_search_request'; -import { i18n } from '@kbn/i18n'; const callWithRequestFactory = (server, request) => { const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); @@ -30,10 +29,6 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { name = 'default'; batchRequestsSupport = true; - label = i18n.translate('tsvb.searchStrategies.default.label', { - defaultMessage: 'Default', - }); - constructor(server) { super(server, callWithRequestFactory, MultiSearchRequest); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index ebc80a5b857eee..e7e6940cb9ae50 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { i18n } from '@kbn/i18n'; import RollupSearchRequest from './rollup_search_request'; import { callWithRequestFactory } from '../call_with_request_factory'; import { AbstractSearchStrategy } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; @@ -11,9 +10,6 @@ import { AbstractSearchStrategy } from '../../../../../../src/legacy/core_plugin export default class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; batchRequestsSupport = false; - label = i18n.translate('tsvb.searchStrategies.rollup.label', { - defaultMessage: 'Rollup', - }); constructor(server) { super(server, callWithRequestFactory, RollupSearchRequest); From 934c1a4d7f1ae62ca0df666f46c808d3694169b0 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Tue, 15 Jan 2019 11:46:52 +0300 Subject: [PATCH 05/58] Changed search by strategy from the last --- .../server/lib/search_strategies/search_strategies_register.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 507fb587a201ca..0797421b9c230f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { findLast } from 'lodash'; import AbstractSearchStrategy from './strategies/abstract_search_strategy'; import DefaultSearchStrategy from './strategies/default_search_strategy'; @@ -38,7 +39,7 @@ export default class SearchStrategiesRegister { } static getStrategyForIndex(indexPattern) { - return strategies.find(searchStrategy => { + return findLast(strategies, searchStrategy => { return searchStrategy.isViable(indexPattern); }); } From bffde5ffa830af4693edf7ae804f3832d8e5a97d Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 15 Jan 2019 13:29:22 +0300 Subject: [PATCH 06/58] add single search request --- .../search_strategies_register.js | 6 +-- .../searh_requests/abstract_request.js | 3 +- .../searh_requests/search_request.js | 37 +++++++++++++++++++ .../searh_requests/single_search_request.js | 34 +++++++++++++++++ .../strategies/abstract_search_strategy.js | 8 ++-- .../strategies/default_search_strategy.js | 6 +-- .../server/lib/vis_data/get_series_data.js | 12 ++---- .../rollup_search_request.js | 4 +- .../rollup_search_strategy.js | 7 +++- 9 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 0797421b9c230f..6025017a6d9b68 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -38,9 +38,7 @@ export default class SearchStrategiesRegister { return searchStrategiesRegister.add(new DefaultSearchStrategy(server)); } - static getStrategyForIndex(indexPattern) { - return findLast(strategies, searchStrategy => { - return searchStrategy.isViable(indexPattern); - }); + static getViableStrategy(req, indexPattern) { + return findLast(strategies, searchStrategy => searchStrategy.isViable(req, indexPattern)); } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js index 0f33be24955da2..2c1c4f707dbe75 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js @@ -17,9 +17,10 @@ * under the License. */ export default class AbstractSearchRequest { - constructor(req, callWithRequest) { + constructor(req, callWithRequest, indexPattern) { this.req = req; this.callWithRequest = callWithRequest; + this.indexPattern = indexPattern; } search() { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js new file mode 100644 index 00000000000000..5347cb144840f0 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchRequest from './abstract_request'; + +import MultiSearchRequest from './multi_search_request'; +import SingleSearchRequest from './single_search_request'; + +export default class SearchRequest extends AbstractSearchRequest { + getSearchRequestType(options) { + const isMultiSearch = Array.isArray(options.body); + const SearchRequest = isMultiSearch ? MultiSearchRequest : SingleSearchRequest; + + return new SearchRequest(this.req, this.callWithRequest, this.indexPattern); + } + + async search(options) { + const concreteSearchRequest = this.getSearchRequestType(options); + + return concreteSearchRequest.search(options); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js new file mode 100644 index 00000000000000..da3e0d48297521 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchRequest from './abstract_request'; + +const SEARCH_METHOD = 'search'; + +export default class MultiSearchRequest extends AbstractSearchRequest { + async search(options) { + const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); + const { responses } = await this.callWithRequest(this.req, SEARCH_METHOD, { + ...options, + index: this.indexPattern, + ignore_throttled: !includeFrozen, + }); + + return responses; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js index b44a04880ca0b3..f2beba86339658 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -18,10 +18,12 @@ */ export default class AbstractSearchStrategy { constructor(server, callWithRequestFactory, SearchRequest) { - this.getSearchRequest = (req) => { - const callWithRequest = callWithRequestFactory(server, req); + this.getCallWithRequestInstance = req => callWithRequestFactory(server, req); - return new SearchRequest(req, callWithRequest); + this.getSearchRequest = (req, indexPattern) => { + const callWithRequest = this.getCallWithRequestInstance(req); + + return new SearchRequest(req, callWithRequest, indexPattern); }; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index 554f8438cc0568..443ff8bcf0981c 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -17,7 +17,7 @@ * under the License. */ import AbstractSearchStrategy from './abstract_search_strategy'; -import MultiSearchRequest from '../searh_requests/multi_search_request'; +import SearchRequest from '../searh_requests/search_request'; const callWithRequestFactory = (server, request) => { const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); @@ -30,10 +30,10 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { batchRequestsSupport = true; constructor(server) { - super(server, callWithRequestFactory, MultiSearchRequest); + super(server, callWithRequestFactory, SearchRequest); } - isViable(indexPattern) { + isViable(req, indexPattern) { return Boolean(indexPattern); } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index faf837db40a4f6..e7650024b4a262 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -21,22 +21,18 @@ import handleResponseBody from './series/handle_response_body'; import handleErrorResponse from './handle_error_response'; import getAnnotations from './get_annotations'; import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; -import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; - export async function getSeriesData(req, panel) { const indexPattern = panel.index_pattern; - const searchStrategy = SearchStrategiesRegister.getStrategyForIndex(indexPattern); - const searchRequest = searchStrategy.getSearchRequest(req); + const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); const body = panel.series .map(series => getRequestParams(req, panel, series, searchStrategy.batchRequestsSupport)) .reduce((acc, items) => acc.concat(items), []); - - // todo: - // try { - return searchRequest.search({ body }, indexPattern) + + return searchRequest.search({ body }) .then(data => { const series = data.map(handleResponseBody(panel, esQueryConfig)); return { diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js index cfafc393097638..f2310740675b24 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js @@ -12,12 +12,12 @@ import { AbstractSearchRequest } from '../../../../../../src/legacy/core_plugins const SEARCH_METHOD = 'rollup.search'; export default class RollupSearchRequest extends AbstractSearchRequest { - async search(options, index) { + async search(options) { const bodies = Array.isArray(options.body) ? options.body : [options.body]; const requests = bodies .map(body => this.callWithRequest(SEARCH_METHOD, { body, - index, + index: this.indexPattern, rest_total_hits_as_int: true, })); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index e7e6940cb9ae50..aebec4d2aa6706 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -15,7 +15,10 @@ export default class RollupSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, RollupSearchRequest); } - isViable(indexPattern) { - return Boolean(indexPattern); + isViable(req, indexPattern) { + const MULTI_INDEX_SEPARATOR = ','; + const splittedIndex = (indexPattern || '').split(MULTI_INDEX_SEPARATOR); + + return Boolean(splittedIndex); } } From 29eda73eda7cc69564184a2de666a36121da6b82 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 15 Jan 2019 15:43:41 +0300 Subject: [PATCH 07/58] rollup_search_strategy add base implementation of isViable method --- .../rollup_search_strategy.js | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index aebec4d2aa6706..3ea499b9883f2b 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -7,6 +7,9 @@ import RollupSearchRequest from './rollup_search_request'; import { callWithRequestFactory } from '../call_with_request_factory'; import { AbstractSearchStrategy } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; +const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; +const INDEX_PATTERN_SEPARATOR = ','; + export default class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; batchRequestsSupport = false; @@ -15,10 +18,19 @@ export default class RollupSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, RollupSearchRequest); } - isViable(req, indexPattern) { - const MULTI_INDEX_SEPARATOR = ','; - const splittedIndex = (indexPattern || '').split(MULTI_INDEX_SEPARATOR); + async isRollupJobExists(req, indexPattern) { + const callWithRequest = this.getCallWithRequestInstance(req); + const requests = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR) + .map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + indexPattern: index, + })); + + return Promise.all(requests) + .then((responses) => responses.some(response => Object.keys(response).length)) + .catch(() => Promise.resolve(false)); + } - return Boolean(splittedIndex); + async isViable(req, indexPattern) { + return await this.isRollupJobExists(req, indexPattern); } } From 51a1507328a00db3b016b6d55c2f6a96253103f1 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 15 Jan 2019 16:54:20 +0300 Subject: [PATCH 08/58] rollup_search_strategy add base implementation of isViable method -fix --- .../search_strategies/search_strategies_register.js | 13 +++++++++---- .../strategies/default_search_strategy.js | 4 ++-- .../metrics/server/lib/vis_data/get_series_data.js | 2 +- .../lib/search_strategies/rollup_search_strategy.js | 13 +++++++------ 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 6025017a6d9b68..43e3395c830300 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -import { findLast } from 'lodash'; import AbstractSearchStrategy from './strategies/abstract_search_strategy'; import DefaultSearchStrategy from './strategies/default_search_strategy'; @@ -25,7 +24,7 @@ const strategies = []; export default class SearchStrategiesRegister { add(searchStrategy) { if (searchStrategy instanceof AbstractSearchStrategy) { - strategies.push(searchStrategy); + strategies.unshift(searchStrategy); } return this; } @@ -38,7 +37,13 @@ export default class SearchStrategiesRegister { return searchStrategiesRegister.add(new DefaultSearchStrategy(server)); } - static getViableStrategy(req, indexPattern) { - return findLast(strategies, searchStrategy => searchStrategy.isViable(req, indexPattern)); + static async getViableStrategy(req, indexPattern) { + for (const searchStrategy of strategies) { + const isViable = await searchStrategy.isViable(req, indexPattern); + + if (isViable) { + return searchStrategy; + } + } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index 443ff8bcf0981c..e0e5c282185392 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -33,7 +33,7 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, SearchRequest); } - isViable(req, indexPattern) { - return Boolean(indexPattern); + isViable() { + return true; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index e7650024b4a262..41ead1da9e5e76 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -24,7 +24,7 @@ import SearchStrategiesRegister from '../search_strategies/search_strategies_reg export async function getSeriesData(req, panel) { const indexPattern = panel.index_pattern; - const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 3ea499b9883f2b..07230d4449082e 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -20,17 +20,18 @@ export default class RollupSearchStrategy extends AbstractSearchStrategy { async isRollupJobExists(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); - const requests = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR) - .map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { - indexPattern: index, - })); + const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); + + const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + indexPattern: index, + })); return Promise.all(requests) - .then((responses) => responses.some(response => Object.keys(response).length)) + .then((responses) => responses.some((response, index) => Boolean(response[indices[index]]))) .catch(() => Promise.resolve(false)); } async isViable(req, indexPattern) { - return await this.isRollupJobExists(req, indexPattern); + return await await this.isRollupJobExists(req, indexPattern); } } From cba722cf9dc6964c32c57f73e313ff8b7c4beeb1 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Tue, 15 Jan 2019 17:10:03 +0300 Subject: [PATCH 09/58] Changed requests due to search request type --- .../searh_requests/single_search_request.js | 5 +- .../server/lib/vis_data/get_annotations.js | 49 ++++++++++++------- .../server/lib/vis_data/get_table_data.js | 31 ++++++------ 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js index da3e0d48297521..95fc0d544abf67 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -23,12 +23,11 @@ const SEARCH_METHOD = 'search'; export default class MultiSearchRequest extends AbstractSearchRequest { async search(options) { const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); - const { responses } = await this.callWithRequest(this.req, SEARCH_METHOD, { + + return await this.callWithRequest(this.req, SEARCH_METHOD, { ...options, index: this.indexPattern, ignore_throttled: !includeFrozen, }); - - return responses; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index cdb08f5289d881..00ac32a4f15c46 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -1,4 +1,4 @@ -/* +* * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright @@ -19,7 +19,7 @@ import buildAnnotationRequest from './build_annotation_request'; import handleAnnotationResponse from './handle_annotation_response'; -import { getIndexPatternObject } from './helpers/get_index_pattern'; +import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; function validAnnotation(annotation) { return annotation.index_pattern && @@ -29,26 +29,34 @@ function validAnnotation(annotation) { annotation.template; } -export default async (req, panel, esQueryConfig) => { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const bodiesPromises = panel.annotations +export default async (req, panel), esQueryConfig => { + const indexPattern = panel.index_pattern; + const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); + const bodies = panel.annotations .filter(validAnnotation) .map(annotation => { - return getAnnotationBody(req, panel, annotation, esQueryConfig); + + const indexPattern = annotation.index_pattern; + const bodies = []; + + bodies.push({ + index: indexPattern, + ignoreUnavailable: true, + }); + + const body = buildAnnotationRequest(req, panel, annotation), esQueryConfig; + body.timeout = '90s'; + bodies.push(body); + return bodies; }); - const bodies = await Promise.all(bodiesPromises); - if (!bodies.length) { - return { - responses: [], - }; - } + + if (!bodies.length) return { responses: [] }; + + const body = bodies.reduce((acc, item) => acc.concat(item), []); + try { - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); - const resp = await callWithRequest(req, 'msearch', { - ignore_throttled: !includeFrozen, - rest_total_hits_as_int: true, - body: bodies.reduce((acc, item) => acc.concat(item), []) - }); + const resp = await searchRequest.search({ body }); const results = {}; panel.annotations .filter(validAnnotation) @@ -61,8 +69,11 @@ export default async (req, panel, esQueryConfig) => { if (error.message === 'missing-indices') return { responses: [] }; throw error; } + }; + +//todo: async function getAnnotationBody(req, panel, annotation, esQueryConfig) { const indexPatternString = annotation.index_pattern; const indexPatternObject = await getIndexPatternObject(req, indexPatternString); @@ -75,4 +86,4 @@ async function getAnnotationBody(req, panel, annotation, esQueryConfig) { }, request, ]; -} \ No newline at end of file +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index a979a6669e74db..c88d20f93c3a42 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -16,28 +16,29 @@ * specific language governing permissions and limitations * under the License. */ -import { get } from 'lodash'; + import buildRequestBody from './table/build_request_body'; import handleErrorResponse from './handle_error_response'; +import { get } from 'lodash'; import processBucket from './table/process_bucket'; -import { getIndexPatternObject } from './helpers/get_index_pattern'; -import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; - +import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; export async function getTableData(req, panel) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); - const indexPatternString = panel.index_pattern; + const indexPattern = panel.index_pattern; + const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); + const body = buildRequestBody(req, panel); + +// todo: +// const indexPatternObject = await getIndexPatternObject(req, indexPatternString); +// const params = { +// index: indexPatternString, +// ignore_throttled: !includeFrozen, +// body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject) +// }; - const esQueryConfig = await getEsQueryConfig(req); - const indexPatternObject = await getIndexPatternObject(req, indexPatternString); - const params = { - index: indexPatternString, - ignore_throttled: !includeFrozen, - body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject) - }; try { - const resp = await callWithRequest(req, 'search', params); + const resp = await searchRequest.search({ body }); const buckets = get(resp, 'aggregations.pivot.buckets', []); return { type: 'table', series: buckets.map(processBucket(panel)) }; } catch (err) { From 6e3cfd2dbd9c533efd444c24b7f0fa997bc160d5 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 15 Jan 2019 17:32:43 +0300 Subject: [PATCH 10/58] refactoring of import Base classes / remove '../../../../../../ --- .../search_strategies_register.js | 10 ++++- .../register_rollup_search_strategy.js | 8 +++- .../rollup_search_request.js | 31 ++++++------- .../rollup_search_strategy.js | 43 +++++++++---------- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 43e3395c830300..faa8333396199b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -17,6 +17,7 @@ * under the License. */ import AbstractSearchStrategy from './strategies/abstract_search_strategy'; +import AbstractSearchRequest from './searh_requests/abstract_request'; import DefaultSearchStrategy from './strategies/default_search_strategy'; const strategies = []; @@ -32,9 +33,14 @@ export default class SearchStrategiesRegister { static init(server) { const searchStrategiesRegister = new SearchStrategiesRegister(); - server.expose('addSearchStrategy', (searchStrategy) => searchStrategiesRegister.add(searchStrategy)); + searchStrategiesRegister.add(new DefaultSearchStrategy(server)); + this.exposeServer(server, searchStrategiesRegister); + } - return searchStrategiesRegister.add(new DefaultSearchStrategy(server)); + static exposeServer(server, searchStrategiesRegister) { + server.expose('addSearchStrategy', (searchStrategy) => searchStrategiesRegister.add(searchStrategy)); + server.expose('AbstractSearchStrategy', AbstractSearchStrategy); + server.expose('AbstractSearchRequest', AbstractSearchRequest); } static async getViableStrategy(req, indexPattern) { diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index cce2cb292e8310..9c29479c4266e9 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -3,12 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import RollupSearchStrategy from './rollup_search_strategy'; +import getRollupSearchStrategy from './rollup_search_strategy'; +import getRollupSearchRequest from './rollup_search_request'; export default (server) => { - const { addSearchStrategy } = server.plugins.metrics; + const { addSearchStrategy, AbstractSearchRequest, AbstractSearchStrategy } = server.plugins.metrics; if (addSearchStrategy) { + const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); + const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest); + addSearchStrategy(new RollupSearchStrategy(server)); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js index f2310740675b24..c14df927fae79e 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js @@ -3,24 +3,19 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -// !!!!!!! -// Question: is it ok? how to make url a little bit more friendly for developers. Can I create alias? -// !!!!!!! -import { AbstractSearchRequest } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; - const SEARCH_METHOD = 'rollup.search'; -export default class RollupSearchRequest extends AbstractSearchRequest { - async search(options) { - const bodies = Array.isArray(options.body) ? options.body : [options.body]; - const requests = bodies - .map(body => this.callWithRequest(SEARCH_METHOD, { - body, - index: this.indexPattern, - rest_total_hits_as_int: true, - })); +export default (AbstractSearchRequest) => + (class RollupSearchRequest extends AbstractSearchRequest { + async search(options) { + const bodies = Array.isArray(options.body) ? options.body : [options.body]; + const requests = bodies + .map(body => this.callWithRequest(SEARCH_METHOD, { + body, + index: this.indexPattern, + rest_total_hits_as_int: true, + })); - return await Promise.all(requests); - } -} + return await Promise.all(requests); + } + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 07230d4449082e..3a9bf9b7d3658b 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,35 +3,34 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import RollupSearchRequest from './rollup_search_request'; import { callWithRequestFactory } from '../call_with_request_factory'; -import { AbstractSearchStrategy } from '../../../../../../src/legacy/core_plugins/metrics/server/lib/search_strategies'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const INDEX_PATTERN_SEPARATOR = ','; -export default class RollupSearchStrategy extends AbstractSearchStrategy { - name = 'rollup'; - batchRequestsSupport = false; +export default (AbstractSearchStrategy, RollupSearchRequest) => + (class RollupSearchStrategy extends AbstractSearchStrategy { + name = 'rollup'; + batchRequestsSupport = false; - constructor(server) { - super(server, callWithRequestFactory, RollupSearchRequest); - } + constructor(server) { + super(server, callWithRequestFactory, RollupSearchRequest); + } - async isRollupJobExists(req, indexPattern) { - const callWithRequest = this.getCallWithRequestInstance(req); - const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); + async isRollupJobExists(req, indexPattern) { + const callWithRequest = this.getCallWithRequestInstance(req); + const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); - const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { - indexPattern: index, - })); + const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + indexPattern: index, + })); - return Promise.all(requests) - .then((responses) => responses.some((response, index) => Boolean(response[indices[index]]))) - .catch(() => Promise.resolve(false)); - } + return Promise.all(requests) + .then((responses) => responses.some((response, index) => Boolean(response[indices[index]]))) + .catch(() => Promise.resolve(false)); + } - async isViable(req, indexPattern) { - return await await this.isRollupJobExists(req, indexPattern); - } -} + async isViable(req, indexPattern) { + return await await this.isRollupJobExists(req, indexPattern); + } + }); From 256e5b183a9ff681ae25a7c982de6a5a2881444a Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 15 Jan 2019 18:20:02 +0300 Subject: [PATCH 11/58] remove extra await --- .../server/lib/search_strategies/rollup_search_strategy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 3a9bf9b7d3658b..0e5b82d42b211a 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -31,6 +31,6 @@ export default (AbstractSearchStrategy, RollupSearchRequest) => } async isViable(req, indexPattern) { - return await await this.isRollupJobExists(req, indexPattern); + return await this.isRollupJobExists(req, indexPattern); } }); From 880a095c05c6d73f6b1558d85106f0875f133429 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 16 Jan 2019 13:32:06 +0300 Subject: [PATCH 12/58] rollup_search_strategy. Refactoring of isRollupJobExists method --- .../rollup_search_strategy.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 0e5b82d42b211a..ea2fa95cab85a4 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -17,7 +17,7 @@ export default (AbstractSearchStrategy, RollupSearchRequest) => super(server, callWithRequestFactory, RollupSearchRequest); } - async isRollupJobExists(req, indexPattern) { + async numberOfRollupJobs(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); @@ -26,11 +26,22 @@ export default (AbstractSearchStrategy, RollupSearchRequest) => })); return Promise.all(requests) - .then((responses) => responses.some((response, index) => Boolean(response[indices[index]]))) - .catch(() => Promise.resolve(false)); + .then((responses) => responses + .reduce((numberOfRollupJobs, response, index) => { + if (response[indices[index]]) { + numberOfRollupJobs += 1; + } + + return numberOfRollupJobs; + }, 0)) + .catch(() => Promise.resolve(0)); + } + + async hasOneRollupJob(req, indexPattern) { + return await this.numberOfRollupJobs(req, indexPattern) === 1; } async isViable(req, indexPattern) { - return await this.isRollupJobExists(req, indexPattern); + return await this.hasOneRollupJob(req, indexPattern); } }); From 64f872edcc6548af7f16490f36c9cb5b1fa520ae Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 16 Jan 2019 13:44:56 +0300 Subject: [PATCH 13/58] remove question --- x-pack/plugins/rollup/index.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index 1fcb732beeed43..ff231bac52c193 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -21,13 +21,6 @@ export function rollup(kibana) { return new kibana.Plugin({ id: PLUGIN.ID, publicDir: resolve(__dirname, 'public'), - - // !!!!!!! - // Question: not sure that it's a good idea to have a reference to 'metrics' from here - // but if we don't add this one we cannot guarantee that 'metrics' plugin was initialized - // - // Maybe we should move 'search strategies' from the 'metrics' folder. - // !!!!!!! require: ['kibana', 'elasticsearch', 'xpack_main', 'metrics'], uiExports: { styleSheetPaths: resolve(__dirname, 'public/index.scss'), From 09e4b54ffa473ba82d37596440773ea749d53c8c Mon Sep 17 00:00:00 2001 From: sulemanof Date: Wed, 16 Jan 2019 14:20:24 +0300 Subject: [PATCH 14/58] Add support of annotations and table data --- .../metrics/server/lib/vis_data/get_annotations.js | 7 ++++--- .../metrics/server/lib/vis_data/get_series_data.js | 3 --- .../metrics/server/lib/vis_data/get_table_data.js | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 00ac32a4f15c46..2d948786b7b97f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -31,7 +31,7 @@ function validAnnotation(annotation) { export default async (req, panel), esQueryConfig => { const indexPattern = panel.index_pattern; - const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const bodies = panel.annotations .filter(validAnnotation) @@ -56,14 +56,15 @@ export default async (req, panel), esQueryConfig => { const body = bodies.reduce((acc, item) => acc.concat(item), []); try { - const resp = await searchRequest.search({ body }); + const responses = await searchRequest.search({ body }); const results = {}; panel.annotations .filter(validAnnotation) .forEach((annotation, index) => { - const data = resp.responses[index]; + const data = responses[index]; results[annotation.id] = handleAnnotationResponse(data, annotation); }); + return results; } catch (error) { if (error.message === 'missing-indices') return { responses: [] }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 41ead1da9e5e76..7e15b1d337a846 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -43,9 +43,6 @@ export async function getSeriesData(req, panel) { }; }) .then(resp => { - // TODO: don't forget to remove - // Should be refactored - return resp; if (!panel.annotations || panel.annotations.length === 0) return resp; return getAnnotations(req, panel, esQueryConfig).then(annotations => { resp[panel.id].annotations = annotations; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index c88d20f93c3a42..54e56d6967a8c4 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -25,7 +25,7 @@ import SearchStrategiesRegister from '../search_strategies/search_strategies_reg export async function getTableData(req, panel) { const indexPattern = panel.index_pattern; - const searchStrategy = SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const body = buildRequestBody(req, panel); From 33c696b2bb949b86eb1163bcc317d79215ef99ed Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 17 Jan 2019 16:17:39 +0300 Subject: [PATCH 15/58] skeleton for adding Search Strategy restrictions --- .../lib/search_strategies/search_strategies_register.js | 7 +++++-- .../strategies/default_search_strategy.js | 7 +++++-- .../metrics/server/lib/vis_data/get_annotations.js | 2 +- .../metrics/server/lib/vis_data/get_series_data.js | 2 +- .../metrics/server/lib/vis_data/get_table_data.js | 2 +- .../lib/search_strategies/rollup_search_strategy.js | 9 +++++++-- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index faa8333396199b..94c28fd8f030cc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -45,10 +45,13 @@ export default class SearchStrategiesRegister { static async getViableStrategy(req, indexPattern) { for (const searchStrategy of strategies) { - const isViable = await searchStrategy.isViable(req, indexPattern); + const { isViable, restrictions } = await searchStrategy.checkForViability(req, indexPattern); if (isViable) { - return searchStrategy; + return { + searchStrategy, + restrictions, + }; } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index e0e5c282185392..362e2622eef398 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -33,7 +33,10 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, SearchRequest); } - isViable() { - return true; + checkForViability() { + return { + isViable: true, + restrictions: {}, + }; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 2d948786b7b97f..d37e5a22b940f6 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -31,7 +31,7 @@ function validAnnotation(annotation) { export default async (req, panel), esQueryConfig => { const indexPattern = panel.index_pattern; - const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const bodies = panel.annotations .filter(validAnnotation) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 7e15b1d337a846..0bd7359a39f4ba 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -24,7 +24,7 @@ import SearchStrategiesRegister from '../search_strategies/search_strategies_reg export async function getSeriesData(req, panel) { const indexPattern = panel.index_pattern; - const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index 54e56d6967a8c4..a1ed051c255ae3 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -25,7 +25,7 @@ import SearchStrategiesRegister from '../search_strategies/search_strategies_reg export async function getTableData(req, panel) { const indexPattern = panel.index_pattern; - const searchStrategy = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const body = buildRequestBody(req, panel); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index ea2fa95cab85a4..35805d334bcff1 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -41,7 +41,12 @@ export default (AbstractSearchStrategy, RollupSearchRequest) => return await this.numberOfRollupJobs(req, indexPattern) === 1; } - async isViable(req, indexPattern) { - return await this.hasOneRollupJob(req, indexPattern); + async checkForViability(req, indexPattern) { + const isViable = await this.hasOneRollupJob(req, indexPattern); + + return { + isViable, + restrictions: {}, + }; } }); From 71454f29578e6aa95ff7bb6e4054cd5783e2a13f Mon Sep 17 00:00:00 2001 From: sulemanof Date: Fri, 18 Jan 2019 11:08:51 +0300 Subject: [PATCH 16/58] Add rollup search capabilities --- .../default_search_capabilities.js | 31 +++++++++++++++++++ .../search_strategies_register.js | 8 +++-- .../strategies/default_search_strategy.js | 7 +++-- .../server/lib/vis_data/get_annotations.js | 12 ++++--- .../server/lib/vis_data/get_series_data.js | 4 +-- .../register_rollup_search_strategy.js | 11 +++++-- .../rollup_search_capabilities.js | 13 ++++++++ .../rollup_search_strategy.js | 30 +++++++----------- 8 files changed, 82 insertions(+), 34 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js new file mode 100644 index 00000000000000..7bec812984d2ef --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export default class DefaultSearchCapabilities { + constructor(req, batchRequestsSupport, capabilities) { + this.request = req; + this.batchRequestsSupport = batchRequestsSupport; + this.capabilities = capabilities; + } + + getTimeZone() { + const { timezone } = this.request.payload.timerange; + + return timezone; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 94c28fd8f030cc..f9dd6037b48451 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -19,6 +19,7 @@ import AbstractSearchStrategy from './strategies/abstract_search_strategy'; import AbstractSearchRequest from './searh_requests/abstract_request'; import DefaultSearchStrategy from './strategies/default_search_strategy'; +import DefaultSearchCapabilities from './default_search_capabilities'; const strategies = []; @@ -38,19 +39,20 @@ export default class SearchStrategiesRegister { } static exposeServer(server, searchStrategiesRegister) { - server.expose('addSearchStrategy', (searchStrategy) => searchStrategiesRegister.add(searchStrategy)); server.expose('AbstractSearchStrategy', AbstractSearchStrategy); server.expose('AbstractSearchRequest', AbstractSearchRequest); + server.expose('DefaultSearchCapabilities', DefaultSearchCapabilities); + server.expose('addSearchStrategy', (searchStrategy) => searchStrategiesRegister.add(searchStrategy)); } static async getViableStrategy(req, indexPattern) { for (const searchStrategy of strategies) { - const { isViable, restrictions } = await searchStrategy.checkForViability(req, indexPattern); + const { isViable, capabilities } = await searchStrategy.checkForViability(req, indexPattern); if (isViable) { return { searchStrategy, - restrictions, + capabilities, }; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index 362e2622eef398..a5f1f8d8a5e663 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -18,25 +18,26 @@ */ import AbstractSearchStrategy from './abstract_search_strategy'; import SearchRequest from '../searh_requests/search_request'; +import DefaultSearchCapabilities from '../default_search_capabilities'; const callWithRequestFactory = (server, request) => { const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); return callWithRequest; }; +const batchRequestsSupport = true; export default class DefaultSearchStrategy extends AbstractSearchStrategy { name = 'default'; - batchRequestsSupport = true; constructor(server) { super(server, callWithRequestFactory, SearchRequest); } - checkForViability() { + checkForViability(req) { return { isViable: true, - restrictions: {}, + capabilities: new DefaultSearchCapabilities(req, batchRequestsSupport) }; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index d37e5a22b940f6..3501a1762c9cd2 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -31,7 +31,7 @@ function validAnnotation(annotation) { export default async (req, panel), esQueryConfig => { const indexPattern = panel.index_pattern; - const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const bodies = panel.annotations .filter(validAnnotation) @@ -40,10 +40,12 @@ export default async (req, panel), esQueryConfig => { const indexPattern = annotation.index_pattern; const bodies = []; - bodies.push({ - index: indexPattern, - ignoreUnavailable: true, - }); + if (capabilities.batchRequestsSupport) { + bodies.push({ + index: indexPattern, + ignoreUnavailable: true, + }); + } const body = buildAnnotationRequest(req, panel, annotation), esQueryConfig; body.timeout = '90s'; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 0bd7359a39f4ba..055817638216fa 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -24,12 +24,12 @@ import SearchStrategiesRegister from '../search_strategies/search_strategies_reg export async function getSeriesData(req, panel) { const indexPattern = panel.index_pattern; - const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); const body = panel.series - .map(series => getRequestParams(req, panel, series, searchStrategy.batchRequestsSupport)) + .map(series => getRequestParams(req, panel, series, capabilities.batchRequestsSupport)) .reduce((acc, items) => acc.concat(items), []); return searchRequest.search({ body }) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index 9c29479c4266e9..63183ea5b804da 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -5,13 +5,20 @@ */ import getRollupSearchStrategy from './rollup_search_strategy'; import getRollupSearchRequest from './rollup_search_request'; +import getRollupSearchCapabilities from './rollup_search_capabilities'; export default (server) => { - const { addSearchStrategy, AbstractSearchRequest, AbstractSearchStrategy } = server.plugins.metrics; + const { + addSearchStrategy, + AbstractSearchRequest, + AbstractSearchStrategy, + DefaultSearchCapabilities, + } = server.plugins.metrics; if (addSearchStrategy) { const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); - const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest); + const RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); + const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); addSearchStrategy(new RollupSearchStrategy(server)); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js new file mode 100644 index 00000000000000..4da9c9598332cb --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -0,0 +1,13 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +export default (DefaultSearchCapabilities) => + (class RollupSearchCapabilities extends DefaultSearchCapabilities { + + getTimeZone() { + // todo: to be refactored + return 'UTC'; + } + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 35805d334bcff1..6d08ded0fce1dd 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,50 +3,42 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const INDEX_PATTERN_SEPARATOR = ','; +const batchRequestsSupport = false; -export default (AbstractSearchStrategy, RollupSearchRequest) => + +export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; - batchRequestsSupport = false; constructor(server) { super(server, callWithRequestFactory, RollupSearchRequest); } - async numberOfRollupJobs(req, indexPattern) { + getAllRollupaCapabilities(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); - const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { indexPattern: index, })); - return Promise.all(requests) - .then((responses) => responses - .reduce((numberOfRollupJobs, response, index) => { - if (response[indices[index]]) { - numberOfRollupJobs += 1; - } - - return numberOfRollupJobs; - }, 0)) - .catch(() => Promise.resolve(0)); + return Promise.all(requests); } - async hasOneRollupJob(req, indexPattern) { - return await this.numberOfRollupJobs(req, indexPattern) === 1; - } async checkForViability(req, indexPattern) { - const isViable = await this.hasOneRollupJob(req, indexPattern); + const rollupCapabilities = await this.getAllRollupaCapabilities(req, indexPattern) + .then((responses) => responses.filter(response => !isEmpty(response))); + + const isViable = rollupCapabilities.length === 1; return { isViable, - restrictions: {}, + capabilities: isViable ? new RollupSearchCapabilities(req, batchRequestsSupport, rollupCapabilities) : null }; } }); From bbda41ff1ea9ed30c61fc0736bb03e6cb7f0268b Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 18 Jan 2019 14:04:10 +0300 Subject: [PATCH 17/58] apply search strategy for annotations request --- .../build_request_body.js} | 5 +- .../annorations/get_request_params.js | 39 +++++++++++++++ .../server/lib/vis_data/get_annotations.js | 48 ++++++------------- .../rollup_search_strategy.js | 5 +- 4 files changed, 57 insertions(+), 40 deletions(-) rename src/legacy/core_plugins/metrics/server/lib/vis_data/{build_annotation_request.js => annorations/build_request_body.js} (89%) create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js similarity index 89% rename from src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js rename to src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js index 1796f3a0a7927b..42690024d533ba 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ - -import buildProcessorFunction from './build_processor_function'; -import processors from './request_processors/annotations'; +import buildProcessorFunction from '../build_processor_function'; +import processors from '../request_processors/annotations'; export default function buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPattern) { const processor = buildProcessorFunction(processors, req, panel, annotation, esQueryConfig, indexPattern); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js new file mode 100644 index 00000000000000..cd583df886d8b1 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import buildRequestBody from './build_request_body'; + +export default (req, panel, annotation, isBatchRequest = true) => { + const bodies = []; + + if (isBatchRequest) { + const indexPattern = annotation.index_pattern; + + bodies.push({ + index: indexPattern, + ignoreUnavailable: true, + }); + } + + bodies.push({ + ...buildRequestBody(req, panel, annotation), + timeout: '90s' + }); + + return bodies; +}; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 3501a1762c9cd2..965244fb9fbf4b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ - -import buildAnnotationRequest from './build_annotation_request'; import handleAnnotationResponse from './handle_annotation_response'; +import getRequestParams from './annorations/get_request_params'; + import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; function validAnnotation(annotation) { @@ -29,53 +29,33 @@ function validAnnotation(annotation) { annotation.template; } -export default async (req, panel), esQueryConfig => { +export default async (req, panel, esQueryConfig) => { const indexPattern = panel.index_pattern; const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); - const bodies = panel.annotations - .filter(validAnnotation) - .map(annotation => { - - const indexPattern = annotation.index_pattern; - const bodies = []; - - if (capabilities.batchRequestsSupport) { - bodies.push({ - index: indexPattern, - ignoreUnavailable: true, - }); - } + const annotations = panel.annotations.filter(validAnnotation); - const body = buildAnnotationRequest(req, panel, annotation), esQueryConfig; - body.timeout = '90s'; - bodies.push(body); - return bodies; - }); + const body = annotations + .map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities.batchRequestsSupport)) + .reduce((acc, item) => acc.concat(item), []); - if (!bodies.length) return { responses: [] }; - - const body = bodies.reduce((acc, item) => acc.concat(item), []); + if (!body.length) return { responses: [] }; try { const responses = await searchRequest.search({ body }); - const results = {}; - panel.annotations - .filter(validAnnotation) - .forEach((annotation, index) => { - const data = responses[index]; - results[annotation.id] = handleAnnotationResponse(data, annotation); - }); - return results; + return annotations + .reduce((acc, annotation, index) => { + acc[annotation.id] = handleAnnotationResponse(responses[index], annotation); + + return acc; + }, {}); } catch (error) { if (error.message === 'missing-indices') return { responses: [] }; throw error; } - }; - //todo: async function getAnnotationBody(req, panel, annotation, esQueryConfig) { const indexPatternString = annotation.index_pattern; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 6d08ded0fce1dd..9e135e2059dc00 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -19,7 +19,7 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil super(server, callWithRequestFactory, RollupSearchRequest); } - getAllRollupaCapabilities(req, indexPattern) { + getAllRollupCapabilities(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { @@ -29,9 +29,8 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil return Promise.all(requests); } - async checkForViability(req, indexPattern) { - const rollupCapabilities = await this.getAllRollupaCapabilities(req, indexPattern) + const rollupCapabilities = await this.getAllRollupCapabilities(req, indexPattern) .then((responses) => responses.filter(response => !isEmpty(response))); const isViable = rollupCapabilities.length === 1; From edf86271ce27df10af1d628e2b2645f1c55d106e Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 18 Jan 2019 18:09:33 +0300 Subject: [PATCH 18/58] set fields capabilities for rollup strategy --- .../default_search_capabilities.js | 4 ++-- .../rollup_search_capabilities.js | 14 ++++++++++++++ .../search_strategies/rollup_search_strategy.js | 17 ++++++++++------- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 7bec812984d2ef..501d72f00184fc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -17,10 +17,10 @@ * under the License. */ export default class DefaultSearchCapabilities { - constructor(req, batchRequestsSupport, capabilities) { + constructor(req, batchRequestsSupport, fieldsCapabilities) { this.request = req; this.batchRequestsSupport = batchRequestsSupport; - this.capabilities = capabilities; + this.fieldsCapabilities = fieldsCapabilities; } getTimeZone() { diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 4da9c9598332cb..a18faf5c35c3fa 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -3,8 +3,22 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { merge } from 'lodash'; + +const toFieldsCapabilities = (rollupData) => { + return Object.keys(rollupData).reduce((capabilities, rollupIndex) => { + return (rollupData[rollupIndex].rollup_jobs || []) + .reduce((acc, job) => merge(acc, job.fields), {}); + }, {}); +}; + export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { + constructor(req, batchRequestsSupport, rollupCapabilities) { + const fieldsCapabilities = toFieldsCapabilities(rollupCapabilities); + + super(req, batchRequestsSupport, fieldsCapabilities); + } getTimeZone() { // todo: to be refactored diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 9e135e2059dc00..49d6d2a8d4fab3 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,14 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty } from 'lodash'; +import { merge } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const INDEX_PATTERN_SEPARATOR = ','; const batchRequestsSupport = false; - export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; @@ -26,18 +25,22 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil indexPattern: index, })); - return Promise.all(requests); + return Promise.all(requests) + .then(data => (data || []).reduce((acc, rollupData) => merge(acc, rollupData), {})); + } + + hasOneRollupJob(rollupCapabilities) { + return Object.keys(rollupCapabilities).length === 1; } async checkForViability(req, indexPattern) { - const rollupCapabilities = await this.getAllRollupCapabilities(req, indexPattern) - .then((responses) => responses.filter(response => !isEmpty(response))); + const rollupCapabilities = await this.getAllRollupCapabilities(req, indexPattern); - const isViable = rollupCapabilities.length === 1; + const isViable = this.hasOneRollupJob(rollupCapabilities); return { isViable, - capabilities: isViable ? new RollupSearchCapabilities(req, batchRequestsSupport, rollupCapabilities) : null + capabilities: isViable ? new RollupSearchCapabilities(req, batchRequestsSupport, rollupCapabilities) : null, }; } }); From cb1cfe79efea1f6881b9c11ab8cdce82fd769e24 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 21 Jan 2019 12:12:46 +0300 Subject: [PATCH 19/58] add timezone, interval into SearchCapabilities --- .../default_search_capabilities.js | 8 ++++-- .../rollup_search_capabilities.js | 28 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 501d72f00184fc..805d935eed3455 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -23,9 +23,11 @@ export default class DefaultSearchCapabilities { this.fieldsCapabilities = fieldsCapabilities; } - getTimeZone() { - const { timezone } = this.request.payload.timerange; + get fixedTimeZone() { + return null; + } - return timezone; + get minimumTimeInterval() { + return null; } } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index a18faf5c35c3fa..417f3fb820d570 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { merge } from 'lodash'; +import { merge, get } from 'lodash'; const toFieldsCapabilities = (rollupData) => { return Object.keys(rollupData).reduce((capabilities, rollupIndex) => { @@ -12,16 +12,36 @@ const toFieldsCapabilities = (rollupData) => { }, {}); }; +const getDateHistogramField = (fieldsCapabilities) => { + let histogramAggregation = null; + + Object.keys(fieldsCapabilities).some((fieldKey) => { + fieldsCapabilities[fieldKey].some((aggregation) => { + if (aggregation.agg === 'date_histogram') { + histogramAggregation = aggregation; + } + return Boolean(histogramAggregation); + }); + return Boolean(histogramAggregation); + }); + return histogramAggregation; +}; + export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { constructor(req, batchRequestsSupport, rollupCapabilities) { const fieldsCapabilities = toFieldsCapabilities(rollupCapabilities); super(req, batchRequestsSupport, fieldsCapabilities); + + this.dateHistogram = getDateHistogramField(fieldsCapabilities); + } + + get fixedTimeZone() { + return get(this.dateHistogram, 'time_zone', null); } - getTimeZone() { - // todo: to be refactored - return 'UTC'; + get minimumTimeInterval() { + return get(this.dateHistogram, 'interval', null); } }); From 9424155e20647f54c2139636a75a4a0c1b58066e Mon Sep 17 00:00:00 2001 From: sulemanof Date: Mon, 21 Jan 2019 16:08:01 +0300 Subject: [PATCH 20/58] Add fields from capabilities --- .../metrics/server/lib/get_fields.js | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/get_fields.js b/src/legacy/core_plugins/metrics/server/lib/get_fields.js index c9588d057614de..6896d635042c0a 100644 --- a/src/legacy/core_plugins/metrics/server/lib/get_fields.js +++ b/src/legacy/core_plugins/metrics/server/lib/get_fields.js @@ -17,11 +17,42 @@ * under the License. */ -import { uniq } from 'lodash'; +import { uniq, forEach } from 'lodash'; +import SearchStrategiesRegister from './search_strategies/search_strategies_register'; + +const FIELD_TYPES = { + STRING: 'string', + DATE: 'date', + NUMBER: 'number' +}; + +const createField = (name, type) => ({ + name, + type, + aggregatable: true, + searchable: true +}); export async function getFields(req) { - const { indexPatternsService } = req.pre; const index = req.query.index || '*'; + const { capabilities } = await SearchStrategiesRegister.getViableStrategy(req, index); + + if (capabilities.fieldsCapabilities) { + const fields = []; + forEach(capabilities.fieldsCapabilities, (aggs, field) => { + if (aggs[0].agg === 'terms') { + fields.push(createField(field, FIELD_TYPES.STRING)); + } else if (aggs[0].agg === 'date_histogram') { + fields.push(createField(field, FIELD_TYPES.DATE)); + } else { + fields.push(createField(field, FIELD_TYPES.NUMBER)); + } + }); + + return fields; + } + + const { indexPatternsService } = req.pre; const resp = await indexPatternsService.getFieldsForWildcard({ pattern: index }); const fields = resp.filter(field => field.aggregatable); return uniq(fields, field => field.name); From 666c7f3b6d92e2dc5116a764ff30ee7104b2e9b3 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 21 Jan 2019 16:08:43 +0300 Subject: [PATCH 21/58] add timezone, interval into SearchCapabilities --- .../default_search_capabilities.js | 33 +++++++++++++++++-- .../annorations/build_request_body.js | 5 +-- .../annorations/get_request_params.js | 6 ++-- .../server/lib/vis_data/get_annotations.js | 19 ++--------- .../server/lib/vis_data/get_series_data.js | 2 +- .../lib/vis_data/helpers/get_bucket_size.js | 7 +++- .../annotations/date_histogram.js | 7 ++-- .../request_processors/annotations/query.js | 4 +-- .../series/date_histogram.js | 6 ++-- .../series/metric_buckets.js | 4 +-- .../series/sibling_buckets.js | 4 +-- .../lib/vis_data/series/build_request_body.js | 5 +-- .../lib/vis_data/series/get_request_params.js | 9 +++-- .../rollup_search_capabilities.js | 17 ++++++++-- 14 files changed, 79 insertions(+), 49 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 805d935eed3455..4a382308b1d0eb 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -16,18 +16,45 @@ * specific language governing permissions and limitations * under the License. */ +import { INTERVAL_STRING_RE } from '../../../common/interval_regexp'; +import unitToSeconds from '../vis_data/helpers/unit_to_seconds'; + +const convertUnitToSeconds = (interval) => { + const matches = interval && interval.match(INTERVAL_STRING_RE); + + return matches ? Number(matches[1]) * unitToSeconds(matches[2]) : 0; +}; + export default class DefaultSearchCapabilities { - constructor(req, batchRequestsSupport, fieldsCapabilities) { - this.request = req; + constructor(request, batchRequestsSupport, fieldsCapabilities) { + this.request = request; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; + + this.validateTimeIntervalRules = []; } get fixedTimeZone() { return null; } - get minimumTimeInterval() { + get defaultTimeInterval() { return null; } + + getSearchTimezone() { + return this.fixedTimeZone || this.request.payload.timerange; + } + + isTimeIntervalValid(intervalString) { + const userInterval = convertUnitToSeconds(intervalString); + const defaultInterval = convertUnitToSeconds(this.defaultTimeInterval); + + return this.validateTimeIntervalRules + .every(validationRule => validationRule(userInterval, defaultInterval)); + } + + getSearchInterval(intervalString) { + return this.isTimeIntervalValid(intervalString) ? intervalString : this.defaultTimeInterval; + } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js index 42690024d533ba..40c9a28563a642 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js @@ -19,8 +19,9 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/annotations'; -export default function buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, annotation, esQueryConfig, indexPattern); +//todo: (req, panel, annotation, esQueryConfig, indexPattern, capabilities +export default function buildAnnotationRequest(...args) { + const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js index cd583df886d8b1..29e64192baed6e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -18,10 +18,10 @@ */ import buildRequestBody from './build_request_body'; -export default (req, panel, annotation, isBatchRequest = true) => { +export default (req, panel, annotation, capabilities) => { const bodies = []; - if (isBatchRequest) { + if (capabilities.batchRequestsSupport) { const indexPattern = annotation.index_pattern; bodies.push({ @@ -31,7 +31,7 @@ export default (req, panel, annotation, isBatchRequest = true) => { } bodies.push({ - ...buildRequestBody(req, panel, annotation), + ...buildRequestBody(req, panel, annotation, capabilities), timeout: '90s' }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 965244fb9fbf4b..8062d72d98d342 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -1,4 +1,4 @@ -* +/* * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright @@ -36,7 +36,7 @@ export default async (req, panel, esQueryConfig) => { const annotations = panel.annotations.filter(validAnnotation); const body = annotations - .map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities.batchRequestsSupport)) + .map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities)) .reduce((acc, item) => acc.concat(item), []); if (!body.length) return { responses: [] }; @@ -55,18 +55,3 @@ export default async (req, panel, esQueryConfig) => { throw error; } }; - -//todo: -async function getAnnotationBody(req, panel, annotation, esQueryConfig) { - const indexPatternString = annotation.index_pattern; - const indexPatternObject = await getIndexPatternObject(req, indexPatternString); - const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject); - request.timeout = '90s'; - return [ - { - index: indexPatternString, - ignoreUnavailable: true, - }, - request, - ]; -} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 055817638216fa..1970fbbb3bd802 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -29,7 +29,7 @@ export async function getSeriesData(req, panel) { const esQueryConfig = await getEsQueryConfig(req); const body = panel.series - .map(series => getRequestParams(req, panel, series, capabilities.batchRequestsSupport)) + .map(series => getRequestParams(req, panel, series, capabilities)) .reduce((acc, items) => acc.concat(items), []); return searchRequest.search({ body }) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js index cd6cf4206d63c4..169905c42c2836 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js @@ -24,7 +24,7 @@ import { INTERVAL_STRING_RE, GTE_INTERVAL_RE } from '../../../../common/interval_regexp'; -export default (req, interval) => { +export default (req, interval, capabilities) => { const from = moment.utc(req.payload.timerange.min); const to = moment.utc(req.payload.timerange.max); const duration = moment.duration(to.valueOf() - from.valueOf(), 'ms'); @@ -44,6 +44,11 @@ export default (req, interval) => { } } + // Todo: 'auto' interval is not working! + if (capabilities) { + interval = capabilities.getSearchInterval(interval); + } + const matches = interval && interval.match(INTERVAL_STRING_RE); if (matches) { bucketSize = Number(matches[1]) * unitToSeconds(matches[2]); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js index 323d5b4911b27f..e426bb8f1dda62 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js @@ -20,12 +20,13 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import getTimerange from '../../helpers/get_timerange'; -export default function dateHistogram(req, panel, annotation) { +export default function dateHistogram(req, panel, annotation, capabilities) { return next => doc => { const timeField = annotation.time_field; - const { bucketSize, intervalString } = getBucketSize(req, 'auto'); + const { bucketSize, intervalString } = getBucketSize(req, 'auto', capabilities); const { from, to } = getTimerange(req); - const { timezone } = req.payload.timerange; + const timezone = capabilities.getSearchTimezone(); + _.set(doc, `aggs.${annotation.id}.date_histogram`, { field: timeField, interval: intervalString, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js index 2e4f765b213213..882f35a2e0ee2f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js @@ -21,10 +21,10 @@ import getBucketSize from '../../helpers/get_bucket_size'; import getTimerange from '../../helpers/get_timerange'; import { buildEsQuery } from '@kbn/es-query'; -export default function query(req, panel, annotation, esQueryConfig, indexPattern) { +export default function query(req, panel, annotation, esQueryConfig, indexPattern, capabilities) { return next => doc => { const timeField = annotation.time_field; - const { bucketSize } = getBucketSize(req, 'auto'); + const { bucketSize } = getBucketSize(req, 'auto', capabilities); const { from, to } = getTimerange(req); doc.size = 0; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js index b760ef20da053d..f75599ded8273b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js @@ -21,12 +21,12 @@ import getBucketSize from '../../helpers/get_bucket_size'; import offsetTime from '../../offset_time'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; import { set } from 'lodash'; -export default function dateHistogram(req, panel, series) { +export default function dateHistogram(req, panel, series, capabilities) { return next => doc => { const { timeField, interval } = getIntervalAndTimefield(panel, series); - const { bucketSize, intervalString } = getBucketSize(req, interval); + const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); const { from, to } = offsetTime(req, series.offset_time); - const { timezone } = req.payload.timerange; + const timezone = capabilities.getSearchTimezone(); set(doc, `aggs.${series.id}.aggs.timeseries.date_histogram`, { field: timeField, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js index ed96bebc9c68cd..764919a46b8e55 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js @@ -21,14 +21,14 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function metricBuckets(req, panel, series) { +export default function metricBuckets(req, panel, series, capabilities) { return next => doc => { const { interval } = getIntervalAndTimefield(panel, series); const { intervalString - } = getBucketSize(req, interval); + } = getBucketSize(req, interval, capabilities); series.metrics .filter(row => !/_bucket$/.test(row.type) && !/^series/.test(row.type)) .forEach(metric => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js index 9f2bc1d4cba0f5..03d61508e61bf6 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js @@ -21,14 +21,14 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function siblingBuckets(req, panel, series) { +export default function siblingBuckets(req, panel, series, capabilities) { return next => doc => { const { interval } = getIntervalAndTimefield(panel, series); const { bucketSize - } = getBucketSize(req, interval); + } = getBucketSize(req, interval, capabilities); series.metrics .filter(row => /_bucket$/.test(row.type)) .forEach(metric => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js index c25127b68a6bcd..7f17e60e235c55 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js @@ -20,8 +20,9 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/series'; -function buildRequestBody(req, panel, series, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, series, esQueryConfig, indexPattern); +//todo: req, panel, series, esQueryConfig, indexPatter, capabilities +function buildRequestBody(...args) { + const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index 23106313e6d728..0c1e6208e9bf3d 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -18,13 +18,12 @@ */ import buildRequestBody from './build_request_body'; -export default (req, panel, series, isBatchRequest = true) => { +export default (req, panel, series, esQueryConfig, capabilities) => { const bodies = []; + const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; const indexPatternObject = await getIndexPatternObject(req, indexPatternString); - if (isBatchRequest) { - const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; - + if (capabilities.batchRequestsSupport) { bodies.push({ index: indexPatternString, ignoreUnavailable: true, @@ -32,7 +31,7 @@ export default (req, panel, series, isBatchRequest = true) => { } bodies.push({ - ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject) + ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject, capabilities) timeout: '90s' }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 417f3fb820d570..5593a72f135f90 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -27,21 +27,32 @@ const getDateHistogramField = (fieldsCapabilities) => { return histogramAggregation; }; +const intervalLessThanMinimum = (userTimeInterval, defaultTimeInterval) => userTimeInterval >= defaultTimeInterval; +const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(userTimeInterval % defaultTimeInterval); + export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { constructor(req, batchRequestsSupport, rollupCapabilities) { const fieldsCapabilities = toFieldsCapabilities(rollupCapabilities); super(req, batchRequestsSupport, fieldsCapabilities); - - this.dateHistogram = getDateHistogramField(fieldsCapabilities); + this.init(); } get fixedTimeZone() { return get(this.dateHistogram, 'time_zone', null); } - get minimumTimeInterval() { + get defaultTimeInterval() { return get(this.dateHistogram, 'interval', null); } + + init() { + this.dateHistogram = getDateHistogramField(this.fieldsCapabilities); + + this.validateTimeIntervalRules = [ + intervalLessThanMinimum, + intervalMultiple, + ]; + } }); From dfbf14eb9b8e4595f45afd1566f5afdb5b45bcd3 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 21 Jan 2019 17:13:33 +0300 Subject: [PATCH 22/58] fix default timezone --- .../server/lib/search_strategies/default_search_capabilities.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 4a382308b1d0eb..d6c3997eed9673 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -43,7 +43,7 @@ export default class DefaultSearchCapabilities { } getSearchTimezone() { - return this.fixedTimeZone || this.request.payload.timerange; + return this.fixedTimeZone || this.request.payload.timerange.timezone; } isTimeIntervalValid(intervalString) { From b75e2666907b8e71582650f6cdf5858a28bb132f Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 21 Jan 2019 19:22:06 +0300 Subject: [PATCH 23/58] Merging of two Rollup Jobs was removed --- .../default_search_capabilities.js | 25 ++++++++++-- .../rollup_search_capabilities.js | 40 +++++-------------- .../rollup_search_strategy.js | 26 ++++++++---- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index d6c3997eed9673..20475469d059d6 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -19,14 +19,18 @@ import { INTERVAL_STRING_RE } from '../../../common/interval_regexp'; import unitToSeconds from '../vis_data/helpers/unit_to_seconds'; -const convertUnitToSeconds = (interval) => { +const convertUnitToSeconds = interval => { const matches = interval && interval.match(INTERVAL_STRING_RE); return matches ? Number(matches[1]) * unitToSeconds(matches[2]) : 0; }; +const getTimezoneFromRequest = request => { + return request.payload.timerange.timezone; +}; + export default class DefaultSearchCapabilities { - constructor(request, batchRequestsSupport, fieldsCapabilities) { + constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { this.request = request; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; @@ -43,7 +47,7 @@ export default class DefaultSearchCapabilities { } getSearchTimezone() { - return this.fixedTimeZone || this.request.payload.timerange.timezone; + return this.fixedTimeZone || getTimezoneFromRequest(this.request); } isTimeIntervalValid(intervalString) { @@ -57,4 +61,19 @@ export default class DefaultSearchCapabilities { getSearchInterval(intervalString) { return this.isTimeIntervalValid(intervalString) ? intervalString : this.defaultTimeInterval; } + + getFieldsByAggregationType(aggregationType) { + const fields = {}; + + Object.keys(this.fieldsCapabilities).forEach(fieldKey => { + this.fieldsCapabilities[fieldKey].some(aggregation => { + if (aggregation.agg === aggregationType) { + fields[fieldKey] = aggregation; + + return true; + } + }); + }); + return fields; + } } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 5593a72f135f90..cb6c8ff5565c25 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -3,39 +3,15 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { merge, get } from 'lodash'; - -const toFieldsCapabilities = (rollupData) => { - return Object.keys(rollupData).reduce((capabilities, rollupIndex) => { - return (rollupData[rollupIndex].rollup_jobs || []) - .reduce((acc, job) => merge(acc, job.fields), {}); - }, {}); -}; - -const getDateHistogramField = (fieldsCapabilities) => { - let histogramAggregation = null; - - Object.keys(fieldsCapabilities).some((fieldKey) => { - fieldsCapabilities[fieldKey].some((aggregation) => { - if (aggregation.agg === 'date_histogram') { - histogramAggregation = aggregation; - } - return Boolean(histogramAggregation); - }); - return Boolean(histogramAggregation); - }); - return histogramAggregation; -}; +import { get } from 'lodash'; -const intervalLessThanMinimum = (userTimeInterval, defaultTimeInterval) => userTimeInterval >= defaultTimeInterval; const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(userTimeInterval % defaultTimeInterval); export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { - constructor(req, batchRequestsSupport, rollupCapabilities) { - const fieldsCapabilities = toFieldsCapabilities(rollupCapabilities); - + constructor(req, batchRequestsSupport, fieldsCapabilities) { super(req, batchRequestsSupport, fieldsCapabilities); + this.init(); } @@ -48,11 +24,17 @@ export default (DefaultSearchCapabilities) => } init() { - this.dateHistogram = getDateHistogramField(this.fieldsCapabilities); + this.dateHistogram = this.getDateHistogramAggregation(); this.validateTimeIntervalRules = [ - intervalLessThanMinimum, intervalMultiple, ]; } + + getDateHistogramAggregation() { + const dateHistogramFields = this.getFieldsByAggregationType('date_histogram'); + const keys = Object.keys(dateHistogramFields); + + return dateHistogramFields[keys[0]]; + } }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 49d6d2a8d4fab3..bd295bc315cc04 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,13 +3,19 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { merge } from 'lodash'; +import { merge, get } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const INDEX_PATTERN_SEPARATOR = ','; const batchRequestsSupport = false; +const getFieldsCapabilities = (rollupData) => { + const rollupIndexKey = Object.keys(rollupData)[0]; + + return get(rollupData, `${rollupIndexKey}.rollup_jobs.[0].fields`); +}; + export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; @@ -18,7 +24,7 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil super(server, callWithRequestFactory, RollupSearchRequest); } - getAllRollupCapabilities(req, indexPattern) { + getRollupData(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { @@ -29,18 +35,24 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil .then(data => (data || []).reduce((acc, rollupData) => merge(acc, rollupData), {})); } - hasOneRollupJob(rollupCapabilities) { - return Object.keys(rollupCapabilities).length === 1; + hasOneRollupIndex(rollupData) { + return Object.keys(rollupData).length === 1; } async checkForViability(req, indexPattern) { - const rollupCapabilities = await this.getAllRollupCapabilities(req, indexPattern); + const rollupData = await this.getRollupData(req, indexPattern); + const isViable = this.hasOneRollupIndex(rollupData); + let capabilities = null; + + if (isViable) { + const fieldsCapabilities = getFieldsCapabilities(rollupData); - const isViable = this.hasOneRollupJob(rollupCapabilities); + capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities); + } return { isViable, - capabilities: isViable ? new RollupSearchCapabilities(req, batchRequestsSupport, rollupCapabilities) : null, + capabilities }; } }); From d99688668d730ad7c287723cdb9c853e877c41dd Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 22 Jan 2019 14:09:03 +0300 Subject: [PATCH 24/58] move getFieldsForWildcard to searchStrategy --- .../metrics/server/lib/get_fields.js | 40 +++---------------- .../server/lib/search_strategies/index.js | 3 -- .../strategies/abstract_search_strategy.js | 8 ++++ .../rollup_search_strategy.js | 10 +++++ 4 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/get_fields.js b/src/legacy/core_plugins/metrics/server/lib/get_fields.js index 6896d635042c0a..41e7f30be96457 100644 --- a/src/legacy/core_plugins/metrics/server/lib/get_fields.js +++ b/src/legacy/core_plugins/metrics/server/lib/get_fields.js @@ -16,44 +16,16 @@ * specific language governing permissions and limitations * under the License. */ - -import { uniq, forEach } from 'lodash'; import SearchStrategiesRegister from './search_strategies/search_strategies_register'; - -const FIELD_TYPES = { - STRING: 'string', - DATE: 'date', - NUMBER: 'number' -}; - -const createField = (name, type) => ({ - name, - type, - aggregatable: true, - searchable: true -}); +import { uniq } from 'lodash'; export async function getFields(req) { - const index = req.query.index || '*'; - const { capabilities } = await SearchStrategiesRegister.getViableStrategy(req, index); - - if (capabilities.fieldsCapabilities) { - const fields = []; - forEach(capabilities.fieldsCapabilities, (aggs, field) => { - if (aggs[0].agg === 'terms') { - fields.push(createField(field, FIELD_TYPES.STRING)); - } else if (aggs[0].agg === 'date_histogram') { - fields.push(createField(field, FIELD_TYPES.DATE)); - } else { - fields.push(createField(field, FIELD_TYPES.NUMBER)); - } - }); + const indexPattern = req.query.index || '*'; + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); - return fields; - } + const fields = (await searchStrategy + .getFieldsForWildcard(req, indexPattern, capabilities)) + .filter(field => field.aggregatable); - const { indexPatternsService } = req.pre; - const resp = await indexPatternsService.getFieldsForWildcard({ pattern: index }); - const fields = resp.filter(field => field.aggregatable); return uniq(fields, field => field.name); } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js index 9042379f42a4f6..ec54845490f921 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js @@ -16,9 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchStrategy from './strategies/abstract_search_strategy'; -import AbstractSearchRequest from './searh_requests/abstract_request'; import SearchStrategiesRegister from './search_strategies_register'; -export { AbstractSearchRequest, AbstractSearchStrategy }; export default SearchStrategiesRegister; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js index f2beba86339658..d11a20e1c45b43 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -26,4 +26,12 @@ export default class AbstractSearchStrategy { return new SearchRequest(req, callWithRequest, indexPattern); }; } + + async getFieldsForWildcard(req, indexPattern) { + const { indexPatternsService } = req.pre; + + return await indexPatternsService.getFieldsForWildcard({ + pattern: indexPattern, + }); + } } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index bd295bc315cc04..c88063a940c145 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -55,4 +55,14 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil capabilities }; } + + async getFieldsForWildcard(req, indexPattern/*, capabilities*/) { + const fields = await super.getFieldsForWildcard(req, indexPattern); + + // TODO: write your logic here + // try to reuse x-pack\plugins\rollup\server\routes\api\index_patterns.js + // probably it make sense to move logic related to merging rollup field to separate file + + return fields; + } }); From ba42e59e6800c8be4fd2d808f38fbc93c0ff4921 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Wed, 16 Jan 2019 16:00:07 +0300 Subject: [PATCH 25/58] Fix TSVB search requests should have a timeout # Conflicts: # src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js --- .../vis_data/helpers/get_es_shard_timeout.js | 22 +++++++++++++++++++ .../server/lib/vis_data/helpers/index.js | 2 ++ 2 files changed, 24 insertions(+) create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js new file mode 100644 index 00000000000000..74d4580fba6003 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export default function getEsShardTimeout(req) { + return `${req.server.config().get('elasticsearch.shardTimeout')}ms`; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js index 01df1a6f54acdc..f5159d8b2bf43b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js @@ -22,6 +22,7 @@ import getAggValue from './get_agg_value'; import getBucketSize from './get_bucket_size'; import getBucketPath from './get_buckets_path'; import getDefaultDecoration from './get_default_decoration'; +import getEsShardTimeout from './get_es_shard_timeout'; import getLastMetric from './get_last_metric'; import getSiblingAggValue from './get_sibling_agg_value'; import getSplits from './get_splits'; @@ -36,6 +37,7 @@ export default { getBucketSize, getBucketPath, getDefaultDecoration, + getEsShardTimeout, getLastMetric, getSiblingAggValue, getSplits, From 2541786fda50a0805cfb3a7fa4689f79bd1530a5 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Wed, 16 Jan 2019 18:27:04 +0300 Subject: [PATCH 26/58] Add unit test --- .../__tests__/helpers/get_es_shard_timeout.js | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js new file mode 100644 index 00000000000000..7d8ab2fb4b2fab --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import sinon from 'sinon'; +import { expect } from 'chai'; +import getEsShardTimeout from '../../helpers/get_es_shard_timeout'; + +describe('getEsShardTimeout', () => { + it('should return the elasticsearch.shardTimeout in ms', () => { + const getConfig = sinon.spy(() => '30000'); + const req = { + server: { + config: () => ({ + get: getConfig + }) + } + }; + const timeout = getEsShardTimeout(req); + + expect(timeout).to.equal('30000ms'); + expect(getConfig.called).to.equal(true); + }); +}); From 0460e3a43ddd04285ae56d2ffea6f9420eb616bb Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 22 Jan 2019 15:41:31 +0300 Subject: [PATCH 27/58] apply getEsShardTimeout for annorations/get_request_params, series/get_request_params --- .../server/lib/vis_data/annorations/get_request_params.js | 4 +++- .../server/lib/vis_data/series/get_request_params.js | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js index 29e64192baed6e..be8e74960ae6d2 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -17,9 +17,11 @@ * under the License. */ import buildRequestBody from './build_request_body'; +import getEsShardTimeout from '../helpers/get_es_shard_timeout'; export default (req, panel, annotation, capabilities) => { const bodies = []; + const timeout = getEsShardTimeout(req); if (capabilities.batchRequestsSupport) { const indexPattern = annotation.index_pattern; @@ -32,7 +34,7 @@ export default (req, panel, annotation, capabilities) => { bodies.push({ ...buildRequestBody(req, panel, annotation, capabilities), - timeout: '90s' + timeout }); return bodies; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index 0c1e6208e9bf3d..8f7cf6e3ac8b4f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -17,11 +17,13 @@ * under the License. */ import buildRequestBody from './build_request_body'; +import getEsShardTimeout from '../helpers/get_es_shard_timeout'; export default (req, panel, series, esQueryConfig, capabilities) => { const bodies = []; const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; const indexPatternObject = await getIndexPatternObject(req, indexPatternString); + const timeout = getEsShardTimeout(req); if (capabilities.batchRequestsSupport) { bodies.push({ @@ -31,8 +33,8 @@ export default (req, panel, series, esQueryConfig, capabilities) => { } bodies.push({ - ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject, capabilities) - timeout: '90s' + ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject, capabilities), + timeout }); return bodies; From e069f65535662552e5d87f47fd1faf6d74c03d0b Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 22 Jan 2019 17:28:49 +0300 Subject: [PATCH 28/58] rename metrics -> tsvb --- src/legacy/core_plugins/metrics/index.js | 2 +- .../search_strategies_register.js | 24 ++++++++----------- x-pack/plugins/rollup/index.js | 2 +- .../register_rollup_search_strategy.js | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/legacy/core_plugins/metrics/index.js b/src/legacy/core_plugins/metrics/index.js index c71cbb6dcfeceb..7f03b6e589b4f0 100644 --- a/src/legacy/core_plugins/metrics/index.js +++ b/src/legacy/core_plugins/metrics/index.js @@ -25,7 +25,7 @@ import SearchStrategiesRegister from './server/lib/search_strategies/search_stra export default function (kibana) { return new kibana.Plugin({ - id: 'metrics', + id: 'tsvb', require: ['kibana', 'elasticsearch'], uiExports: { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index f9dd6037b48451..8dc333b254fa40 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -24,25 +24,21 @@ import DefaultSearchCapabilities from './default_search_capabilities'; const strategies = []; export default class SearchStrategiesRegister { - add(searchStrategy) { - if (searchStrategy instanceof AbstractSearchStrategy) { - strategies.unshift(searchStrategy); - } - return this; - } - static init(server) { - const searchStrategiesRegister = new SearchStrategiesRegister(); - - searchStrategiesRegister.add(new DefaultSearchStrategy(server)); - this.exposeServer(server, searchStrategiesRegister); - } + SearchStrategiesRegister.add(new DefaultSearchStrategy(server)); - static exposeServer(server, searchStrategiesRegister) { server.expose('AbstractSearchStrategy', AbstractSearchStrategy); server.expose('AbstractSearchRequest', AbstractSearchRequest); server.expose('DefaultSearchCapabilities', DefaultSearchCapabilities); - server.expose('addSearchStrategy', (searchStrategy) => searchStrategiesRegister.add(searchStrategy)); + + server.expose('addSearchStrategy', (searchStrategy) => SearchStrategiesRegister.add(searchStrategy)); + } + + static add(searchStrategy) { + if (searchStrategy instanceof AbstractSearchStrategy) { + strategies.unshift(searchStrategy); + } + return this; } static async getViableStrategy(req, indexPattern) { diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index ff231bac52c193..2d082ca1116c47 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -21,7 +21,7 @@ export function rollup(kibana) { return new kibana.Plugin({ id: PLUGIN.ID, publicDir: resolve(__dirname, 'public'), - require: ['kibana', 'elasticsearch', 'xpack_main', 'metrics'], + require: ['kibana', 'elasticsearch', 'xpack_main', 'tsvb'], uiExports: { styleSheetPaths: resolve(__dirname, 'public/index.scss'), managementSections: [ diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index 63183ea5b804da..bdc154938f3681 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -13,7 +13,7 @@ export default (server) => { AbstractSearchRequest, AbstractSearchStrategy, DefaultSearchCapabilities, - } = server.plugins.metrics; + } = server.plugins.tsvb; if (addSearchStrategy) { const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); From 5ef7a841e29b7fb94bb91767c7c673c2617530ec Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 23 Jan 2019 13:50:31 +0300 Subject: [PATCH 29/58] search_strategies_register refactoring: move 'add' method from class --- .../search_strategies_register.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index 8dc333b254fa40..e8e3b8ee3119c1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -23,22 +23,21 @@ import DefaultSearchCapabilities from './default_search_capabilities'; const strategies = []; +const addStrategy = searchStrategy => { + if (searchStrategy instanceof AbstractSearchStrategy) { + strategies.unshift(searchStrategy); + } + return this; +}; + export default class SearchStrategiesRegister { static init(server) { - SearchStrategiesRegister.add(new DefaultSearchStrategy(server)); - server.expose('AbstractSearchStrategy', AbstractSearchStrategy); server.expose('AbstractSearchRequest', AbstractSearchRequest); server.expose('DefaultSearchCapabilities', DefaultSearchCapabilities); + server.expose('addSearchStrategy', searchStrategy => addStrategy(searchStrategy)); - server.expose('addSearchStrategy', (searchStrategy) => SearchStrategiesRegister.add(searchStrategy)); - } - - static add(searchStrategy) { - if (searchStrategy instanceof AbstractSearchStrategy) { - strategies.unshift(searchStrategy); - } - return this; + addStrategy(new DefaultSearchStrategy(server)); } static async getViableStrategy(req, indexPattern) { From a50a67eec44bf22bb62058de4e69b475838a795e Mon Sep 17 00:00:00 2001 From: sulemanof Date: Wed, 23 Jan 2019 14:20:01 +0300 Subject: [PATCH 30/58] Add merge rollup capabilities with fields --- .../default_search_capabilities.js | 18 +---- .../strategies/default_search_strategy.js | 4 +- .../lib/merge_capabilities_with_fields.js | 69 +++++++++++++++++++ .../rollup_search_capabilities.js | 10 +-- .../rollup_search_strategy.js | 24 +++---- .../server/routes/api/index_patterns.js | 58 +--------------- 6 files changed, 90 insertions(+), 93 deletions(-) create mode 100644 x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 20475469d059d6..be0ff7be13d398 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -30,8 +30,9 @@ const getTimezoneFromRequest = request => { }; export default class DefaultSearchCapabilities { - constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { + constructor(request, indexPattern, batchRequestsSupport, fieldsCapabilities = {}) { this.request = request; + this.indexPattern = indexPattern; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; @@ -61,19 +62,4 @@ export default class DefaultSearchCapabilities { getSearchInterval(intervalString) { return this.isTimeIntervalValid(intervalString) ? intervalString : this.defaultTimeInterval; } - - getFieldsByAggregationType(aggregationType) { - const fields = {}; - - Object.keys(this.fieldsCapabilities).forEach(fieldKey => { - this.fieldsCapabilities[fieldKey].some(aggregation => { - if (aggregation.agg === aggregationType) { - fields[fieldKey] = aggregation; - - return true; - } - }); - }); - return fields; - } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index a5f1f8d8a5e663..a6b8193fdd19fc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -34,10 +34,10 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, SearchRequest); } - checkForViability(req) { + checkForViability(req, indexPattern) { return { isViable: true, - capabilities: new DefaultSearchCapabilities(req, batchRequestsSupport) + capabilities: new DefaultSearchCapabilities(req, indexPattern, batchRequestsSupport) }; } } diff --git a/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js new file mode 100644 index 00000000000000..fe4541e676975a --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js @@ -0,0 +1,69 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ + +// Merge rollup capabilities information with field information + +const mergeCapabilitiesWithFields = (rollupIndexCapabilities, fieldsFromFieldCapsApi, previousFields = []) => { + const rollupFields = [...previousFields]; + const rollupFieldNames = []; + + Object.keys(rollupIndexCapabilities).forEach(agg => { + + // Field names of the aggregation + const fields = Object.keys(rollupIndexCapabilities[agg]); + + // Default field information + const defaultField = { + name: null, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }; + + // Date histogram agg only ever has one field defined, let date type overwrite a + // previous type if defined (such as number from max and min aggs). + if(agg === 'date_histogram') { + const timeFieldName = fields[0]; + const fieldCapsKey = `${timeFieldName}.${agg}.timestamp`; + const newField = { + ...fieldsFromFieldCapsApi[fieldCapsKey], + ...defaultField, + name: timeFieldName, + }; + const existingField = rollupFields.find(field => field.name === timeFieldName); + + if(existingField) { + Object.assign(existingField, newField); + } else { + rollupFieldNames.push(timeFieldName); + rollupFields.push(newField); + } + } + // For all other aggs, filter out ones that have already been added to the field list + // because the same field can be part of multiple aggregations, but end consumption + // doesn't differentiate fields based on their aggregation abilities. + else { + rollupFields.push( + ...fields + .filter(field => !rollupFieldNames.includes(field)) + .map(field => { + // Expand each field into object format that end consumption expects. + const fieldCapsKey = `${field}.${agg}.value`; + rollupFieldNames.push(field); + return { + ...fieldsFromFieldCapsApi[fieldCapsKey], + ...defaultField, + name: field, + }; + }) + ); + } + }); + + return rollupFields; +}; + +export default mergeCapabilitiesWithFields; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index cb6c8ff5565c25..685db48c75c3e8 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -9,8 +9,8 @@ const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(use export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { - constructor(req, batchRequestsSupport, fieldsCapabilities) { - super(req, batchRequestsSupport, fieldsCapabilities); + constructor(req, indexPattern, batchRequestsSupport, fieldsCapabilities) { + super(req, indexPattern, batchRequestsSupport, fieldsCapabilities); this.init(); } @@ -32,9 +32,9 @@ export default (DefaultSearchCapabilities) => } getDateHistogramAggregation() { - const dateHistogramFields = this.getFieldsByAggregationType('date_histogram'); - const keys = Object.keys(dateHistogramFields); + const dateHistogramField = this.fieldsCapabilities[this.indexPattern].aggs.date_histogram; - return dateHistogramFields[keys[0]]; + // there is also only one valid date_histogram field + return Object.values(dateHistogramField)[0]; } }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index c88063a940c145..b004cdd45d8e99 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,19 +3,15 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { merge, get } from 'lodash'; +import { merge, indexBy } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; +import mergeCapabilitiesWithFields from '../merge_capabilities_with_fields'; +import { getCapabilitiesForRollupIndices } from '../map_capabilities'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const INDEX_PATTERN_SEPARATOR = ','; const batchRequestsSupport = false; -const getFieldsCapabilities = (rollupData) => { - const rollupIndexKey = Object.keys(rollupData)[0]; - - return get(rollupData, `${rollupIndexKey}.rollup_jobs.[0].fields`); -}; - export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; @@ -45,9 +41,9 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil let capabilities = null; if (isViable) { - const fieldsCapabilities = getFieldsCapabilities(rollupData); + const fieldsCapabilities = getCapabilitiesForRollupIndices(rollupData); - capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities); + capabilities = new RollupSearchCapabilities(req, indexPattern, batchRequestsSupport, fieldsCapabilities); } return { @@ -56,13 +52,11 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil }; } - async getFieldsForWildcard(req, indexPattern/*, capabilities*/) { + async getFieldsForWildcard(req, indexPattern, { fieldsCapabilities }) { const fields = await super.getFieldsForWildcard(req, indexPattern); + const fieldsFromFieldCapsApi = indexBy(fields, 'name'); + const rollupIndexCapabilities = fieldsCapabilities[indexPattern].aggs; - // TODO: write your logic here - // try to reuse x-pack\plugins\rollup\server\routes\api\index_patterns.js - // probably it make sense to move logic related to merging rollup field to separate file - - return fields; + return mergeCapabilitiesWithFields(rollupIndexCapabilities, fieldsFromFieldCapsApi); } }); diff --git a/x-pack/plugins/rollup/server/routes/api/index_patterns.js b/x-pack/plugins/rollup/server/routes/api/index_patterns.js index f79b127df98faf..2c2ae186252481 100644 --- a/x-pack/plugins/rollup/server/routes/api/index_patterns.js +++ b/x-pack/plugins/rollup/server/routes/api/index_patterns.js @@ -10,6 +10,7 @@ import { wrapEsError, wrapUnknownError } from '../../lib/error_wrappers'; import { licensePreRoutingFactory } from'../../lib/license_pre_routing_factory'; import indexBy from 'lodash/collection/indexBy'; import { getCapabilitiesForRollupIndices } from '../../lib/map_capabilities'; +import mergeCapabilitiesWithFields from '../../lib/merge_capabilities_with_fields'; import querystring from 'querystring'; /** @@ -57,7 +58,6 @@ export function registerFieldsForWildcardRoute(server) { const callWithRequest = callWithRequestFactory(server, request); const rollupFields = []; - const rollupFieldNames = []; const fieldsFromFieldCapsApi = indexBy(fields, 'name'); const rollupIndexCapabilities = getCapabilitiesForRollupIndices(await callWithRequest('rollup.rollupIndexCapabilities', { indexPattern: rollupIndex @@ -66,62 +66,10 @@ export function registerFieldsForWildcardRoute(server) { // Keep meta fields metaFields.forEach(field => fieldsFromFieldCapsApi[field] && rollupFields.push(fieldsFromFieldCapsApi[field])); - // Merge rollup capabilities information with field information - Object.keys(rollupIndexCapabilities).forEach(agg => { - - // Field names of the aggregation - const fields = Object.keys(rollupIndexCapabilities[agg]); - - // Default field information - const defaultField = { - name: null, - searchable: true, - aggregatable: true, - readFromDocValues: true, - }; - - // Date histogram agg only ever has one field defined, let date type overwrite a - // previous type if defined (such as number from max and min aggs). - if(agg === 'date_histogram') { - const timeFieldName = fields[0]; - const fieldCapsKey = `${timeFieldName}.${agg}.timestamp`; - const newField = { - ...fieldsFromFieldCapsApi[fieldCapsKey], - ...defaultField, - name: timeFieldName, - }; - const existingField = rollupFields.find(field => field.name === timeFieldName); - - if(existingField) { - Object.assign(existingField, newField); - } else { - rollupFieldNames.push(timeFieldName); - rollupFields.push(newField); - } - } - // For all other aggs, filter out ones that have already been added to the field list - // because the same field can be part of multiple aggregations, but end consumption - // doesn't differentiate fields based on their aggregation abilities. - else { - rollupFields.push( - ...fields - .filter(field => !rollupFieldNames.includes(field)) - .map(field => { - // Expand each field into object format that end consumption expects. - const fieldCapsKey = `${field}.${agg}.value`; - rollupFieldNames.push(field); - return { - ...fieldsFromFieldCapsApi[fieldCapsKey], - ...defaultField, - name: field, - }; - }) - ); - } - }); + const mergedRollupFields = mergeCapabilitiesWithFields(rollupIndexCapabilities, fieldsFromFieldCapsApi, rollupFields); return { - fields: rollupFields + fields: [ ...rollupFields, ...mergedRollupFields ] }; } catch(err) { if (isEsError(err)) { From 69ba70dfc4bc671fab4e51ecda740877b2dccb1c Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 23 Jan 2019 19:01:54 +0300 Subject: [PATCH 31/58] Add merge rollup capabilities with fields - small fixes --- .../default_search_capabilities.js | 3 +- .../strategies/default_search_strategy.js | 4 +- .../rollup_search_capabilities.js | 7 ++-- .../rollup_search_strategy.js | 39 +++++++++---------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index be0ff7be13d398..1cd5ebe1219109 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -30,9 +30,8 @@ const getTimezoneFromRequest = request => { }; export default class DefaultSearchCapabilities { - constructor(request, indexPattern, batchRequestsSupport, fieldsCapabilities = {}) { + constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { this.request = request; - this.indexPattern = indexPattern; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index a6b8193fdd19fc..a5f1f8d8a5e663 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -34,10 +34,10 @@ export default class DefaultSearchStrategy extends AbstractSearchStrategy { super(server, callWithRequestFactory, SearchRequest); } - checkForViability(req, indexPattern) { + checkForViability(req) { return { isViable: true, - capabilities: new DefaultSearchCapabilities(req, indexPattern, batchRequestsSupport) + capabilities: new DefaultSearchCapabilities(req, batchRequestsSupport) }; } } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 685db48c75c3e8..d5a94cd2018ce0 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -9,9 +9,10 @@ const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(use export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { - constructor(req, indexPattern, batchRequestsSupport, fieldsCapabilities) { - super(req, indexPattern, batchRequestsSupport, fieldsCapabilities); + constructor(req, batchRequestsSupport, fieldsCapabilities, rollupIndex) { + super(req, batchRequestsSupport, fieldsCapabilities); + this.rollupIndex = rollupIndex; this.init(); } @@ -32,7 +33,7 @@ export default (DefaultSearchCapabilities) => } getDateHistogramAggregation() { - const dateHistogramField = this.fieldsCapabilities[this.indexPattern].aggs.date_histogram; + const dateHistogramField = this.fieldsCapabilities[this.rollupIndex].aggs.date_histogram; // there is also only one valid date_histogram field return Object.values(dateHistogramField)[0]; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index b004cdd45d8e99..05545732c9ca57 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,15 +3,17 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { merge, indexBy } from 'lodash'; +import { indexBy } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; import mergeCapabilitiesWithFields from '../merge_capabilities_with_fields'; import { getCapabilitiesForRollupIndices } from '../map_capabilities'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; -const INDEX_PATTERN_SEPARATOR = ','; +const DEFAULT_INDEX_PATTERN = '*'; const batchRequestsSupport = false; +const getRollupIndices = rollupData => Object.keys(rollupData); + export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; @@ -20,42 +22,37 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil super(server, callWithRequestFactory, RollupSearchRequest); } - getRollupData(req, indexPattern) { + async getRollupData(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); - const indices = (indexPattern || '').split(INDEX_PATTERN_SEPARATOR); - const requests = indices.map(index => callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { - indexPattern: index, - })); - - return Promise.all(requests) - .then(data => (data || []).reduce((acc, rollupData) => merge(acc, rollupData), {})); + return await callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + indexPattern, + }); } - hasOneRollupIndex(rollupData) { - return Object.keys(rollupData).length === 1; - } - - async checkForViability(req, indexPattern) { + async checkForViability(req, indexPattern = DEFAULT_INDEX_PATTERN) { const rollupData = await this.getRollupData(req, indexPattern); - const isViable = this.hasOneRollupIndex(rollupData); + const rollupIndices = getRollupIndices(rollupData); + const isViable = rollupIndices.length === 1 && indexPattern !== DEFAULT_INDEX_PATTERN; let capabilities = null; if (isViable) { + const [rollupIndex] = rollupIndices; const fieldsCapabilities = getCapabilitiesForRollupIndices(rollupData); - capabilities = new RollupSearchCapabilities(req, indexPattern, batchRequestsSupport, fieldsCapabilities); + capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities, rollupIndex); } return { isViable, - capabilities + capabilities, }; } - async getFieldsForWildcard(req, indexPattern, { fieldsCapabilities }) { - const fields = await super.getFieldsForWildcard(req, indexPattern); + async getFieldsForWildcard(req, indexPattern, { fieldsCapabilities, rollupIndex }) { + const fields = await super.getFieldsForWildcard(req, indexPattern); + const fieldsFromFieldCapsApi = indexBy(fields, 'name'); - const rollupIndexCapabilities = fieldsCapabilities[indexPattern].aggs; + const rollupIndexCapabilities = fieldsCapabilities[rollupIndex].aggs; return mergeCapabilitiesWithFields(rollupIndexCapabilities, fieldsFromFieldCapsApi); } From c2cfb01d036d91699788e8fc1c84faa2725f6f6e Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 24 Jan 2019 17:33:04 +0300 Subject: [PATCH 32/58] Add support of 'Everything' aggregation for Rollup Search --- .../server/lib/vis_data/helpers/get_splits.js | 23 +++++++---- .../series/date_histogram.js | 3 +- .../request_processors/series/index.js | 4 +- .../series/normalize_query.js | 39 +++++++++++++++++++ .../response_processors/series/math.js | 4 +- .../response_processors/series/percentile.js | 4 +- .../series/std_deviation_bands.js | 4 +- .../series/std_deviation_sibling.js | 4 +- .../response_processors/series/std_metric.js | 4 +- .../response_processors/series/std_sibling.js | 4 +- .../vis_data/series/handle_response_body.js | 8 ++-- 11 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js index 99734f7f131c3c..bdcca1f69a4a20 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js @@ -23,8 +23,16 @@ import _ from 'lodash'; import getLastMetric from './get_last_metric'; import getSplitColors from './get_split_colors'; import { formatKey } from './format_key'; -export default function getSplits(resp, panel, series) { - const meta = _.get(resp, `aggregations.${series.id}.meta`); + +const getTimeSeries = (resp, series) => + _.get(resp, `aggregations.timeseries`) || + _.get(resp, `aggregations.${series.id}.timeseries`); + +export default function getSplits(resp, panel, series, meta) { + if (!meta) { + meta = _.get(resp, `aggregations.${series.id}.meta`); + } + const color = new Color(series.color); const metric = getLastMetric(series); if (_.has(resp, `aggregations.${series.id}.buckets`)) { @@ -41,7 +49,7 @@ export default function getSplits(resp, panel, series) { }); } - if(series.split_mode === 'filters' && _.isPlainObject(buckets)) { + if (series.split_mode === 'filters' && _.isPlainObject(buckets)) { return series.split_filters.map(filter => { const bucket = _.get(resp, `aggregations.${series.id}.buckets.${filter.id}`); bucket.id = `${series.id}:${filter.id}`; @@ -54,9 +62,10 @@ export default function getSplits(resp, panel, series) { } } - const timeseries = _.get(resp, `aggregations.${series.id}.timeseries`); + const timeseries = getTimeSeries(resp, series); + const mergeObj = { - timeseries + timeseries, }; series.metrics .filter(m => /_bucket/.test(m.type)) @@ -69,8 +78,8 @@ export default function getSplits(resp, panel, series) { label: series.label || calculateLabel(metric, series.metrics), color: color.string(), ...mergeObj, - meta - } + meta, + }, ]; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js index f75599ded8273b..5dd613be3e8057 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js @@ -41,7 +41,8 @@ export default function dateHistogram(req, panel, series, capabilities) { set(doc, `aggs.${series.id}.meta`, { timeField, intervalString, - bucketSize + bucketSize, + seriesId: series.id, }); return next(doc); }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js index 8008a63e102209..02e084bc779a19 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js @@ -26,6 +26,7 @@ import dateHistogram from './date_histogram'; import metricBuckets from './metric_buckets'; import siblingBuckets from './sibling_buckets'; import filterRatios from './filter_ratios'; +import normalizeQuery from './normalize_query'; export default [ query, @@ -36,5 +37,6 @@ export default [ dateHistogram, metricBuckets, siblingBuckets, - filterRatios + filterRatios, + normalizeQuery ]; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js new file mode 100644 index 00000000000000..2683cd9b5c74e3 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +const { set, get, isEmpty } = require('lodash'); + +const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); + +function removeEmptyTopLevelAggregation(doc, series) { + const filter = get(doc, `aggs.${series.id}.filter`); + + if (isEmptyFilter(filter)) { + const meta = get(doc, `aggs.${series.id}.meta`); + set(doc, `aggs`, doc.aggs[series.id].aggs); + set(doc, `aggs.timeseries.meta`, meta); + } + + return doc; +} + +export default function normalizeQuery(req, panel, series) { + return next => doc => { + return next(removeEmptyTopLevelAggregation(doc, series)); + }; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js index 0fa97bfb109e11..ada43f1e35cf5a 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js @@ -25,7 +25,7 @@ import getSplits from '../../helpers/get_splits'; import mapBucket from '../../helpers/map_bucket'; import { evaluate } from 'tinymath'; -export function mathAgg(resp, panel, series) { +export function mathAgg(resp, panel, series, meta) { return next => results => { const mathMetric = last(series.metrics); if (mathMetric.type !== 'math') return next(results); @@ -38,7 +38,7 @@ export function mathAgg(resp, panel, series) { return true; }); const decoration = getDefaultDecoration(series); - const splits = getSplits(resp, panel, series); + const splits = getSplits(resp, panel, series, meta); const mathSeries = splits.map(split => { if (mathMetric.variables.length) { // Gather the data for the splits. The data will either be a sibling agg or diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js index c04b880dd03ac8..a1bf05baadc645 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js @@ -22,12 +22,12 @@ import getAggValue from '../../helpers/get_agg_value'; import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; -export default function percentile(resp, panel, series) { +export default function percentile(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type !== 'percentile') return next(results); - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { metric.percentiles.forEach(percentile => { const label = (split.label) + ` (${percentile.value})`; const data = split.timeseries.buckets.map(bucket => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js index 7a52ef26e72ba6..5d4e6991fe524b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js @@ -21,11 +21,11 @@ import _ from 'lodash'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import mapBucket from '../../helpers/map_bucket'; -export default function stdDeviationBands(resp, panel, series) { +export default function stdDeviationBands(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type === 'std_deviation' && metric.mode === 'band') { - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const upper = split.timeseries.buckets.map(mapBucket(_.assign({}, metric, { mode: 'upper' }))); const lower = split.timeseries.buckets.map(mapBucket(_.assign({}, metric, { mode: 'lower' }))); results.push({ diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js index db26760e30fde8..45f5a0eb6bf73a 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js @@ -21,11 +21,11 @@ import _ from 'lodash'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import getSiblingAggValue from '../../helpers/get_sibling_agg_value'; -export default function stdDeviationSibling(resp, panel, series) { +export default function stdDeviationSibling(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.mode === 'band' && metric.type === 'std_deviation_bucket') { - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const mapBucketByMode = (mode) => { return bucket => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js index bc3cf9dab6338c..06030a09f25100 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js @@ -21,7 +21,7 @@ import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import mapBucket from '../../helpers/map_bucket'; -export default function stdMetric(resp, panel, series) { +export default function stdMetric(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type === 'std_deviation' && metric.mode === 'band') { @@ -32,7 +32,7 @@ export default function stdMetric(resp, panel, series) { } if (/_bucket$/.test(metric.type)) return next(results); const decoration = getDefaultDecoration(series); - getSplits(resp, panel, series).forEach(split => { + getSplits(resp, panel, series, meta).forEach(split => { const data = split.timeseries.buckets.map(mapBucket(metric)); results.push({ id: `${split.id}`, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js index eac35d459b4a13..c3784e96e73908 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js @@ -21,7 +21,7 @@ import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import getSiblingAggValue from '../../helpers/get_sibling_agg_value'; -export default function stdSibling(resp, panel, series) { +export default function stdSibling(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); @@ -29,7 +29,7 @@ export default function stdSibling(resp, panel, series) { if (metric.type === 'std_deviation_bucket' && metric.mode === 'band') return next(results); const decoration = getDefaultDecoration(series); - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const data = split.timeseries.buckets.map(bucket => { return [bucket.key, getSiblingAggValue(split, metric)]; }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js index f732db82f45edd..12fc07598ad4cc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js @@ -44,9 +44,11 @@ export default function handleResponseBody(panel) { }) ); } - const seriesId = keys[0]; - const series = panel.series.find(s => s.id === seriesId); - const processor = buildProcessorFunction(processors, resp, panel, series); + const [ seriesId ] = keys; + const { meta } = resp.aggregations[seriesId]; + const series = panel.series.find(s => s.id === meta.seriesId); + const processor = buildProcessorFunction(processors, resp, panel, series, meta); + return processor([]); }; } From 3f5508efa6f5df252d337a2779af677301ca2120 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Thu, 24 Jan 2019 18:42:26 +0300 Subject: [PATCH 33/58] Return back metrics plugin --- src/legacy/core_plugins/metrics/index.js | 1 - x-pack/plugins/rollup/index.js | 2 +- .../lib/search_strategies/register_rollup_search_strategy.js | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/legacy/core_plugins/metrics/index.js b/src/legacy/core_plugins/metrics/index.js index 7f03b6e589b4f0..358628e56d480f 100644 --- a/src/legacy/core_plugins/metrics/index.js +++ b/src/legacy/core_plugins/metrics/index.js @@ -25,7 +25,6 @@ import SearchStrategiesRegister from './server/lib/search_strategies/search_stra export default function (kibana) { return new kibana.Plugin({ - id: 'tsvb', require: ['kibana', 'elasticsearch'], uiExports: { diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index 2d082ca1116c47..ff231bac52c193 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -21,7 +21,7 @@ export function rollup(kibana) { return new kibana.Plugin({ id: PLUGIN.ID, publicDir: resolve(__dirname, 'public'), - require: ['kibana', 'elasticsearch', 'xpack_main', 'tsvb'], + require: ['kibana', 'elasticsearch', 'xpack_main', 'metrics'], uiExports: { styleSheetPaths: resolve(__dirname, 'public/index.scss'), managementSections: [ diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index bdc154938f3681..63183ea5b804da 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -13,7 +13,7 @@ export default (server) => { AbstractSearchRequest, AbstractSearchStrategy, DefaultSearchCapabilities, - } = server.plugins.tsvb; + } = server.plugins.metrics; if (addSearchStrategy) { const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); From 18e79ba16d083e8b4949249216e7d94a7281f465 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 25 Jan 2019 17:53:17 +0300 Subject: [PATCH 34/58] remove 'metrics' from the X-pack\rollup require --- x-pack/plugins/rollup/index.js | 5 +++-- .../register_rollup_search_strategy.js | 20 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index ff231bac52c193..edd79511d72ba5 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -21,7 +21,7 @@ export function rollup(kibana) { return new kibana.Plugin({ id: PLUGIN.ID, publicDir: resolve(__dirname, 'public'), - require: ['kibana', 'elasticsearch', 'xpack_main', 'metrics'], + require: ['kibana', 'elasticsearch', 'xpack_main'], uiExports: { styleSheetPaths: resolve(__dirname, 'public/index.scss'), managementSections: [ @@ -61,7 +61,8 @@ export function rollup(kibana) { ) { server.plugins.index_management.addIndexManagementDataEnricher(rollupDataEnricher); } - registerRollupSearchStrategy(server); + + registerRollupSearchStrategy(this.kbnServer, server); } }); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index 63183ea5b804da..8dc1e4d154cc13 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -7,7 +7,11 @@ import getRollupSearchStrategy from './rollup_search_strategy'; import getRollupSearchRequest from './rollup_search_request'; import getRollupSearchCapabilities from './rollup_search_capabilities'; -export default (server) => { +export default (kbnServer, server) => kbnServer.afterPluginsInit(() => { + if (!server.plugins.metrics) { + return; + } + const { addSearchStrategy, AbstractSearchRequest, @@ -15,13 +19,9 @@ export default (server) => { DefaultSearchCapabilities, } = server.plugins.metrics; - if (addSearchStrategy) { - const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); - const RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); - const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); - - addSearchStrategy(new RollupSearchStrategy(server)); - } + const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); + const RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); + const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); - return server; -}; + addSearchStrategy(new RollupSearchStrategy(server)); +}); From b4290f7b690c138dc81fccbdb68f1ddfa1951c78 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Mon, 28 Jan 2019 14:24:40 +0300 Subject: [PATCH 35/58] Fix test cases --- .../series/__tests__/date_histogram.js | 20 ++-- .../series/__tests__/build_request_body.js | 99 +++++++++---------- 2 files changed, 62 insertions(+), 57 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js index 3d77d98e56950a..7c28b84de22eb4 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js @@ -20,12 +20,14 @@ import dateHistogram from '../date_histogram'; import { expect } from 'chai'; import sinon from 'sinon'; +import DefaultSearchCapabilities from '../../../../search_strategies/default_search_capabilities'; describe('dateHistogram(req, panel, series)', () => { let panel; let series; let req; + let capabilities; beforeEach(() => { req = { payload: { @@ -42,17 +44,18 @@ describe('dateHistogram(req, panel, series)', () => { interval: '10s' }; series = { id: 'test' }; + capabilities = new DefaultSearchCapabilities(req, true); }); it('calls next when finished', () => { const next = sinon.spy(); - dateHistogram(req, panel, series)(next)({}); + dateHistogram(req, panel, series, capabilities)(next)({}); expect(next.calledOnce).to.equal(true); }); it('returns valid date histogram', () => { const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -73,7 +76,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 10, intervalString: '10s', - timeField: '@timestamp' + timeField: '@timestamp', + seriesId: 'test' } } } @@ -83,7 +87,7 @@ describe('dateHistogram(req, panel, series)', () => { it('returns valid date histogram (offset by 1h)', () => { series.offset_time = '1h'; const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -104,7 +108,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 10, intervalString: '10s', - timeField: '@timestamp' + timeField: '@timestamp', + seriesId: 'test' } } } @@ -117,7 +122,7 @@ describe('dateHistogram(req, panel, series)', () => { series.series_time_field = 'timestamp'; series.series_interval = '20s'; const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -138,7 +143,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 20, intervalString: '20s', - timeField: 'timestamp' + timeField: 'timestamp', + seriesId: 'test' } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index cb944bb8363fae..b6d820ed30cade 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -75,24 +75,40 @@ const body = JSON.parse(` } `); -import buildRequestBody from '../build_request_body'; +import sinon from 'sinon'; import { expect } from 'chai'; +import buildRequestBody from '../build_request_body'; describe('buildRequestBody(req)', () => { it('returns a valid body', () => { const panel = body.panels[0]; const series = panel.series[0]; - const config = { + const getSearchTimezone = sinon.spy(() => 'UTC'); + const getSearchInterval = sinon.spy(); + const capabilities = { + getSearchTimezone, + getSearchInterval + }; + const config = { allowLeadingWildcards: true, queryStringOptions: {}, }; const doc = buildRequestBody({ payload: body }, panel, series, config); + expect(getSearchTimezone.calledOnce).to.equal(true); expect(doc).to.eql({ size: 0, query: { bool: { - filter: [], must: [ + { + range: { + '@timestamp': { + gte: 1485463055881, + lte: 1485463955881, + format: 'epoch_millis' + } + } + }, { bool: { must: [ @@ -105,58 +121,41 @@ describe('buildRequestBody(req)', () => { ], must_not: [] } - }, - { - range: { - '@timestamp': { - gte: 1485463055881, - lte: 1485463955881, - format: 'epoch_millis' - } - } - }, - ], - must_not: [], - should: [], + } + ] } }, - aggs: { - 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7': { - filter: { - match_all: {} - }, - meta: { - timeField: '@timestamp', - bucketSize: 10, - intervalString: '10s' - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - interval: '10s', - min_doc_count: 0, - time_zone: 'UTC', - extended_bounds: { - min: 1485463055881, - max: 1485463955881 - } - }, - aggs: { - 'c9b5f9c1-e403-11e6-be91-6f7688e9fac7': { - bucket_script: { - buckets_path: { - count: '_count' - }, - script: { - source: 'count * 1', - lang: 'expression' - }, - gap_policy: 'skip' - } + 'aggs': { + 'timeseries': { + 'aggs': { + 'c9b5f9c1-e403-11e6-be91-6f7688e9fac7': { + 'bucket_script': { + 'buckets_path': { + 'count': '_count' + }, + 'gap_policy': 'skip', + 'script': { + 'lang': 'expression', + 'source': 'count * 1' } } } + }, + 'date_histogram': { + 'extended_bounds': { + 'max': 1485463955881, + 'min': 1485463055881 + }, + 'field': '@timestamp', + 'interval': '10s', + 'min_doc_count': 0, + 'time_zone': 'UTC' + }, + 'meta': { + 'bucketSize': 10, + 'intervalString': '10s', + 'seriesId': 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7', + 'timeField': '@timestamp' } } } From 2a855ed4b9d57b41457245001c725481389ba81d Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 28 Jan 2019 15:30:06 +0300 Subject: [PATCH 36/58] fix broken test: fail: "apis InfraOps GraphQL Endpoints metrics should basically work" --- .../rollup_search_strategy.js | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 05545732c9ca57..654a72462161b1 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { indexBy } from 'lodash'; +import { indexBy, isString } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; import mergeCapabilitiesWithFields from '../merge_capabilities_with_fields'; import { getCapabilitiesForRollupIndices } from '../map_capabilities'; @@ -13,6 +13,7 @@ const DEFAULT_INDEX_PATTERN = '*'; const batchRequestsSupport = false; const getRollupIndices = rollupData => Object.keys(rollupData); +const isIndexPatternValid = indexPattern => isString(indexPattern) && indexPattern !== DEFAULT_INDEX_PATTERN; export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { @@ -30,16 +31,21 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil } async checkForViability(req, indexPattern = DEFAULT_INDEX_PATTERN) { - const rollupData = await this.getRollupData(req, indexPattern); - const rollupIndices = getRollupIndices(rollupData); - const isViable = rollupIndices.length === 1 && indexPattern !== DEFAULT_INDEX_PATTERN; + let isViable = false; let capabilities = null; - if (isViable) { - const [rollupIndex] = rollupIndices; - const fieldsCapabilities = getCapabilitiesForRollupIndices(rollupData); + if (isIndexPatternValid(indexPattern)) { + const rollupData = await this.getRollupData(req, indexPattern); + const rollupIndices = getRollupIndices(rollupData); - capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities, rollupIndex); + isViable = rollupIndices.length === 1; + + if (isViable) { + const [rollupIndex] = rollupIndices; + const fieldsCapabilities = getCapabilitiesForRollupIndices(rollupData); + + capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities, rollupIndex); + } } return { From 705ee14fe9578ebc9ec61bdcb87dbe4a0d8a0600 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 28 Jan 2019 16:45:12 +0300 Subject: [PATCH 37/58] rollup search - split by terms is not working --- .../server/lib/vis_data/series/handle_response_body.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js index 12fc07598ad4cc..cd108139fffe78 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js @@ -45,8 +45,8 @@ export default function handleResponseBody(panel) { ); } const [ seriesId ] = keys; - const { meta } = resp.aggregations[seriesId]; - const series = panel.series.find(s => s.id === meta.seriesId); + const meta = get(resp, `aggregations.${seriesId}.meta`, {}); + const series = panel.series.find(s => s.id === (meta.seriesId || seriesId)); const processor = buildProcessorFunction(processors, resp, panel, series, meta); return processor([]); From 60cc12a1adbfdc8269f5b9a0e9e2b3926d5c0b60 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Tue, 29 Jan 2019 13:19:42 +0300 Subject: [PATCH 38/58] Add count metric --- .../__tests__/helpers/get_agg_value.js | 9 +++ .../lib/vis_data/helpers/get_agg_value.js | 66 +++++++++---------- ...xtended_stats_types.js => metric_types.js} | 9 ++- 3 files changed, 48 insertions(+), 36 deletions(-) rename src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/{extended_stats_types.js => metric_types.js} (82%) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js index 804d50464b47a3..8b6d194c92aad4 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js @@ -131,6 +131,15 @@ describe('getAggValue', () => { }, }; + + describe('count', () => { + testAgg( + basicWithDerv, + { id: 'test', type: 'count' }, + 2 + ); + }); + describe('derivative', () => { testAgg( basicWithDerv, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js index 57a58cf00cfe4f..5bf32ee4ff174b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js @@ -18,7 +18,7 @@ */ import { get, includes, max, min, sum } from 'lodash'; -import extendStatsTypes from './extended_stats_types'; +import { EXTENDED_STATS_TYPES, METRIC_TYPES } from './metric_types'; const aggFns = { max, @@ -29,7 +29,7 @@ const aggFns = { export default (row, metric) => { // Extended Stats - if (includes(extendStatsTypes, metric.type)) { + if (includes(EXTENDED_STATS_TYPES, metric.type)) { const isStdDeviation = /^std_deviation/.test(metric.type); const modeIsBounds = ~['upper', 'lower'].indexOf(metric.mode); if (isStdDeviation && modeIsBounds) { @@ -38,38 +38,36 @@ export default (row, metric) => { return get(row, `${metric.id}.${metric.type}`); } - // Percentiles - if (metric.type === 'percentile') { - let percentileKey = `${metric.percent}`; - if (!/\./.test(`${metric.percent}`)) { - percentileKey = `${metric.percent}.0`; - } - return row[metric.id].values[percentileKey]; - } - - if (metric.type === 'percentile_rank') { - const percentileRankKey = `${metric.value}`; - return ( - row[metric.id] && - row[metric.id].values && - row[metric.id].values[percentileRankKey] - ); - } + switch (metric.type) { + case METRIC_TYPES.PERCENTILE: + let percentileKey = `${metric.percent}`; + if (!/\./.test(`${metric.percent}`)) { + percentileKey = `${metric.percent}.0`; + } + return row[metric.id].values[percentileKey]; + case METRIC_TYPES.PERCENTILE_RANK: + const percentileRankKey = `${metric.value}`; + return ( + row[metric.id] && + row[metric.id].values && + row[metric.id].values[percentileRankKey] + ); + case METRIC_TYPES.TOP_HIT: + if (row[metric.id].doc_count === 0) return null; + const hits = get(row, [metric.id, 'docs', 'hits', 'hits'], []); + const values = hits.map(doc => { + return get(doc, `_source.${metric.field}`, 0); + }); + const aggWith = (metric.agg_with && aggFns[metric.agg_with]) || aggFns.avg; + return aggWith(values); + case METRIC_TYPES.COUNT: + return get(row, 'doc_count', null); + default: + // Derivatives + const normalizedValue = get(row, `${metric.id}.normalized_value`, null); - if (metric.type === 'top_hit') { - if (row[metric.id].doc_count === 0) return null; - const hits = get(row, [metric.id, 'docs', 'hits', 'hits'], []); - const values = hits.map(doc => { - return get(doc, `_source.${metric.field}`, 0); - }); - const aggWith = (metric.agg_with && aggFns[metric.agg_with]) || aggFns.avg; - return aggWith(values); + // Everything else + const value = get(row, `${metric.id}.value`, null); + return normalizedValue || value; } - - // Derivatives - const normalizedValue = get(row, `${metric.id}.normalized_value`, null); - - // Everything else - const value = get(row, `${metric.id}.value`, null); - return normalizedValue || value; }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js similarity index 82% rename from src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js rename to src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js index e206a1ce5c23e8..1143291141dc4e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js @@ -17,10 +17,15 @@ * under the License. */ -export default [ +export const EXTENDED_STATS_TYPES = [ 'std_deviation', 'variance', 'sum_of_squares' ]; - +export const METRIC_TYPES = { + PERCENTILE: 'percentile', + PERCENTILE_RANK: 'percentile_rank', + TOP_HIT: 'top_hit', + COUNT: 'count' +}; From eb3b3dbd9b30971848511641d13a697eb05c81ba Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 29 Jan 2019 14:11:45 +0300 Subject: [PATCH 39/58] /get_bucket_size.js. Add support of 'auto' interval, Add support of gte intervals e.g.: >=1m --- .../default_search_capabilities.js | 16 ++++-- .../lib/vis_data/helpers/get_bucket_size.js | 57 ++++++++++++------- .../rollup_search_capabilities.js | 11 ++++ 3 files changed, 58 insertions(+), 26 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 1cd5ebe1219109..334b3e210c7f04 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -34,7 +34,6 @@ export default class DefaultSearchCapabilities { this.request = request; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; - this.validateTimeIntervalRules = []; } @@ -46,19 +45,26 @@ export default class DefaultSearchCapabilities { return null; } + get defaultTimeIntervalInSeconds() { + return this.getIntervalInSeconds(this.defaultTimeInterval); + } + getSearchTimezone() { return this.fixedTimeZone || getTimezoneFromRequest(this.request); } + getIntervalInSeconds(intervalString) { + return Boolean(intervalString) ? convertUnitToSeconds(intervalString) : 0; + } + isTimeIntervalValid(intervalString) { - const userInterval = convertUnitToSeconds(intervalString); - const defaultInterval = convertUnitToSeconds(this.defaultTimeInterval); + const userInterval = this.getIntervalInSeconds(intervalString); return this.validateTimeIntervalRules - .every(validationRule => validationRule(userInterval, defaultInterval)); + .every(validationRule => validationRule(userInterval, this.defaultTimeIntervalInSeconds)); } - getSearchInterval(intervalString) { + getValidTimeInterval(intervalString) { return this.isTimeIntervalValid(intervalString) ? intervalString : this.defaultTimeInterval; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js index 169905c42c2836..687cc02853038d 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js @@ -22,38 +22,53 @@ import moment from 'moment'; import unitToSeconds from './unit_to_seconds'; import { INTERVAL_STRING_RE, - GTE_INTERVAL_RE + GTE_INTERVAL_RE, } from '../../../../common/interval_regexp'; -export default (req, interval, capabilities) => { - const from = moment.utc(req.payload.timerange.min); - const to = moment.utc(req.payload.timerange.max); + +const calculateBucketData = (timeInterval, capabilities) => { + const intervalString = capabilities ? capabilities.getValidTimeInterval(timeInterval) : timeInterval; + const intervalStringMatch = intervalString.match(INTERVAL_STRING_RE); + + let bucketSize = Number(intervalStringMatch[1]) * unitToSeconds(intervalStringMatch[2]); + + // don't go too small + if (bucketSize < 1) { + bucketSize = 1; + } + + return { + bucketSize, + intervalString, + }; +}; + +const getTimeRangeBucketSize = ({ min, max }) => { + const from = moment.utc(min); + const to = moment.utc(max); const duration = moment.duration(to.valueOf() - from.valueOf(), 'ms'); - let bucketSize = calculateAuto.near(100, duration).asSeconds(); - if (bucketSize < 1) bucketSize = 1; // don't go too small + + return calculateAuto.near(100, duration).asSeconds(); +}; + +export default (req, interval, capabilities) => { + const bucketSize = getTimeRangeBucketSize(req.payload.timerange); let intervalString = `${bucketSize}s`; - const gteAutoMatch = interval && interval.match(GTE_INTERVAL_RE); + const gteAutoMatch = Boolean(interval) && interval.match(GTE_INTERVAL_RE); + if (gteAutoMatch) { - const intervalStringMatch = gteAutoMatch[1].match(INTERVAL_STRING_RE); - const gteBucketSize = Number(intervalStringMatch[1]) * unitToSeconds(intervalStringMatch[2]); - if (gteBucketSize >= bucketSize) { - return { - bucketSize: gteBucketSize, - intervalString: gteAutoMatch[1] - }; - } - } + const bucketData = calculateBucketData(gteAutoMatch[1], capabilities); - // Todo: 'auto' interval is not working! - if (capabilities) { - interval = capabilities.getSearchInterval(interval); + if (bucketData.bucketSize >= bucketSize) { + return bucketData; + } } const matches = interval && interval.match(INTERVAL_STRING_RE); + if (matches) { - bucketSize = Number(matches[1]) * unitToSeconds(matches[2]); intervalString = interval; } - return { bucketSize, intervalString }; + return calculateBucketData(intervalString, capabilities); }; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index d5a94cd2018ce0..4eafe215aff662 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -6,6 +6,7 @@ import { get } from 'lodash'; const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(userTimeInterval % defaultTimeInterval); +const roundN = (num, base) => Math.ceil(num / base) * base; export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { @@ -38,4 +39,14 @@ export default (DefaultSearchCapabilities) => // there is also only one valid date_histogram field return Object.values(dateHistogramField)[0]; } + + getValidTimeInterval(intervalString) { + if (this.isTimeIntervalValid(intervalString)) { + return intervalString; + } + + const userInterval = this.getIntervalInSeconds(intervalString); + + return `${roundN(userInterval, this.defaultTimeIntervalInSeconds)}s`; + } }); From cc3cda4466e1959c0646cbfaadfa659590894bd3 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 29 Jan 2019 15:04:05 +0300 Subject: [PATCH 40/58] fix build_request_body test --- .../lib/vis_data/series/__tests__/build_request_body.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index b6d820ed30cade..8aa160cf6cfbb8 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -85,9 +85,12 @@ describe('buildRequestBody(req)', () => { const series = panel.series[0]; const getSearchTimezone = sinon.spy(() => 'UTC'); const getSearchInterval = sinon.spy(); + const getValidTimeInterval = sinon.spy(() => '10s'); + const capabilities = { getSearchTimezone, - getSearchInterval + getSearchInterval, + getValidTimeInterval }; const config = { allowLeadingWildcards: true, From 1540961ffec9addf3365af305d885f65f9133b1c Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 30 Jan 2019 14:10:22 +0300 Subject: [PATCH 41/58] [Rollup] [Phase 1] Error handling - rollup search errors should be more user friendly --- .../core_plugins/metrics/public/components/_error.scss | 1 + .../core_plugins/metrics/public/components/error.js | 8 +++++++- .../metrics/public/components/visualization.js | 4 ++-- .../lib/search_strategies/rollup_search_strategy.js | 7 ++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/legacy/core_plugins/metrics/public/components/_error.scss b/src/legacy/core_plugins/metrics/public/components/_error.scss index b9905c10113632..8c17278dca6b8a 100644 --- a/src/legacy/core_plugins/metrics/public/components/_error.scss +++ b/src/legacy/core_plugins/metrics/public/components/_error.scss @@ -16,6 +16,7 @@ } } +.tvbError__title, .tvbError__additional, .tvbError__stack { margin-top: $euiSizeS; diff --git a/src/legacy/core_plugins/metrics/public/components/error.js b/src/legacy/core_plugins/metrics/public/components/error.js index eef0a6d841705c..f4c054190fe7e5 100644 --- a/src/legacy/core_plugins/metrics/public/components/error.js +++ b/src/legacy/core_plugins/metrics/public/components/error.js @@ -22,10 +22,12 @@ import React from 'react'; import _ from 'lodash'; import { FormattedMessage } from '@kbn/i18n/react'; +const guidPattern = /\[[[a-f\d-\\]{36}\]/g; + function ErrorComponent(props) { const { error } = props; let additionalInfo; - const type = _.get(error, 'error.caused_by.type'); + const type = _.get(error, 'error.caused_by.type') || _.get(error, 'error.type'); let reason = _.get(error, 'error.caused_by.reason'); const title = _.get(error, 'error.caused_by.title'); @@ -33,6 +35,10 @@ function ErrorComponent(props) { reason = _.get(error, 'message'); } + if (['runtime_exception', 'illegal_argument_exception'].includes(type)) { + reason = _.get(error, 'error.reason').replace(guidPattern, ``); + } + if (type === 'script_exception') { const scriptStack = _.get(error, 'error.caused_by.script_stack'); reason = _.get(error, 'error.caused_by.caused_by.reason'); diff --git a/src/legacy/core_plugins/metrics/public/components/visualization.js b/src/legacy/core_plugins/metrics/public/components/visualization.js index 505809afdd8001..4095b3424e20f1 100644 --- a/src/legacy/core_plugins/metrics/public/components/visualization.js +++ b/src/legacy/core_plugins/metrics/public/components/visualization.js @@ -27,7 +27,7 @@ import topN from './vis_types/top_n/vis'; import table from './vis_types/table/vis'; import gauge from './vis_types/gauge/vis'; import markdown from './vis_types/markdown/vis'; -import Error from './error'; +import ErrorComponent from './error'; import NoData from './no_data'; const types = { @@ -46,7 +46,7 @@ function Visualization(props) { if (error) { return (
- +
); } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 654a72462161b1..5dcec141f2cf9f 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -23,11 +23,12 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil super(server, callWithRequestFactory, RollupSearchRequest); } - async getRollupData(req, indexPattern) { + getRollupData(req, indexPattern) { const callWithRequest = this.getCallWithRequestInstance(req); - return await callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + + return callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { indexPattern, - }); + }).catch(() => Promise.resolve({})); } async checkForViability(req, indexPattern = DEFAULT_INDEX_PATTERN) { From 265650304e10bb0726155d88b092b17f866da2e3 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 30 Jan 2019 17:48:25 +0300 Subject: [PATCH 42/58] [Rollup] [Phase 1] Table View - research the query to ES - sorting is not wokring --- .../loader/pipeline_helpers/build_pipeline.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts index 2bff967c09829e..62d2212587a1e0 100644 --- a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts +++ b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts @@ -49,7 +49,7 @@ interface Schemas { [key: string]: any[] | undefined; } -type buildVisFunction = (visState: VisState, schemas: Schemas) => string; +type buildVisFunction = (visState: VisState, schemas: Schemas, uiState: any) => string; interface BuildPipelineVisFunction { [key: string]: buildVisFunction; @@ -201,8 +201,11 @@ export const buildPipelineVisFunction: BuildPipelineVisFunction = { input_control_vis: visState => { return `input_control_vis ${prepareJson('visConfig', visState.params)}`; }, - metrics: visState => { - return `tsvb ${prepareJson('params', visState.params)}`; + metrics: (visState, schemas, uiState) => { + const params = prepareJson('params', visState.params); + const uiState = prepareJson('uiState', uiState); + + return `tsvb ${params} ${uiState}`; }, timelion: visState => { const expression = prepareString('expression', visState.params.expression); @@ -300,6 +303,7 @@ export const buildPipeline = ( const query = searchSource.getField('query'); const filters = searchSource.getField('filter'); const visState = vis.getCurrentState(); + const uiState = vis.getUiState(); // context let pipeline = `kibana | kibana_context `; @@ -325,7 +329,7 @@ export const buildPipeline = ( const schemas = getSchemas(vis, params.timeRange); if (buildPipelineVisFunction[vis.type.name]) { - pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas); + pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas, uiState); } else if (vislibCharts.includes(vis.type.name)) { const visConfig = visState.params; visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); From e9650ae50be40c03c6977715748d5136dcf86764 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 31 Jan 2019 14:56:33 +0300 Subject: [PATCH 43/58] Merge #26006 into rollup # Conflicts: # src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js # src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js --- .../annorations/get_request_params.js | 9 ++- .../server/lib/vis_data/get_annotations.js | 11 +-- .../server/lib/vis_data/get_series_data.js | 9 ++- .../server/lib/vis_data/get_table_data.js | 16 ++-- .../annotations/date_histogram.js | 2 +- .../series/__tests__/date_histogram.js | 16 +++- .../series/date_histogram.js | 2 +- .../series/metric_buckets.js | 2 +- .../series/sibling_buckets.js | 2 +- .../series/__tests__/build_request_body.js | 79 ++++++++++--------- .../lib/vis_data/series/build_request_body.js | 1 - .../lib/vis_data/series/get_request_params.js | 11 +-- .../lib/vis_data/table/build_request_body.js | 4 +- 13 files changed, 84 insertions(+), 80 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js index be8e74960ae6d2..f3cfd066494899 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -18,14 +18,15 @@ */ import buildRequestBody from './build_request_body'; import getEsShardTimeout from '../helpers/get_es_shard_timeout'; +import { getIndexPatternObject } from '../helpers/get_index_pattern'; -export default (req, panel, annotation, capabilities) => { +export default async (req, panel, annotation, esQueryConfig, capabilities) => { const bodies = []; const timeout = getEsShardTimeout(req); + const indexPattern = annotation.index_pattern; + const indexPatternObject = await getIndexPatternObject(req, indexPattern); if (capabilities.batchRequestsSupport) { - const indexPattern = annotation.index_pattern; - bodies.push({ index: indexPattern, ignoreUnavailable: true, @@ -33,7 +34,7 @@ export default (req, panel, annotation, capabilities) => { } bodies.push({ - ...buildRequestBody(req, panel, annotation, capabilities), + ...buildRequestBody(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities), timeout }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 8062d72d98d342..4c47f93aee1b75 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -19,8 +19,6 @@ import handleAnnotationResponse from './handle_annotation_response'; import getRequestParams from './annorations/get_request_params'; -import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; - function validAnnotation(annotation) { return annotation.index_pattern && annotation.time_field && @@ -29,15 +27,14 @@ function validAnnotation(annotation) { annotation.template; } -export default async (req, panel, esQueryConfig) => { +export default async (req, panel, esQueryConfig, searchStrategy, capabilities) => { const indexPattern = panel.index_pattern; - const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const annotations = panel.annotations.filter(validAnnotation); - const body = annotations - .map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities)) - .reduce((acc, item) => acc.concat(item), []); + const bodiesPromises = annotations.map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities)); + const body = (await Promise.all(bodiesPromises)) + .reduce((acc, items) => acc.concat(items), []); if (!body.length) return { responses: [] }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 1970fbbb3bd802..ff697452bcf9b0 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -21,6 +21,7 @@ import handleResponseBody from './series/handle_response_body'; import handleErrorResponse from './handle_error_response'; import getAnnotations from './get_annotations'; import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; +import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; export async function getSeriesData(req, panel) { const indexPattern = panel.index_pattern; @@ -28,13 +29,13 @@ export async function getSeriesData(req, panel) { const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); - const body = panel.series - .map(series => getRequestParams(req, panel, series, capabilities)) + const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig, capabilities)); + const body = (await Promise.all(bodiesPromises)) .reduce((acc, items) => acc.concat(items), []); return searchRequest.search({ body }) .then(data => { - const series = data.map(handleResponseBody(panel, esQueryConfig)); + const series = data.map(handleResponseBody(panel)); return { [panel.id]: { id: panel.id, @@ -44,7 +45,7 @@ export async function getSeriesData(req, panel) { }) .then(resp => { if (!panel.annotations || panel.annotations.length === 0) return resp; - return getAnnotations(req, panel, esQueryConfig).then(annotations => { + return getAnnotations(req, panel, esQueryConfig, searchStrategy, capabilities).then(annotations => { resp[panel.id].annotations = annotations; return resp; }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index a1ed051c255ae3..692d0df75199c7 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -22,20 +22,16 @@ import handleErrorResponse from './handle_error_response'; import { get } from 'lodash'; import processBucket from './table/process_bucket'; import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; +import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; +import { getIndexPatternObject } from './helpers/get_index_pattern'; export async function getTableData(req, panel) { const indexPattern = panel.index_pattern; - const { searchStrategy } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); - const body = buildRequestBody(req, panel); - -// todo: -// const indexPatternObject = await getIndexPatternObject(req, indexPatternString); -// const params = { -// index: indexPatternString, -// ignore_throttled: !includeFrozen, -// body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject) -// }; + const esQueryConfig = await getEsQueryConfig(req); + const indexPatternObject = await getIndexPatternObject(req, indexPattern); + const body = buildRequestBody(req, panel, esQueryConfig, indexPatternObject, capabilities); try { const resp = await searchRequest.search({ body }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js index e426bb8f1dda62..e9a488db23f7e3 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js @@ -20,7 +20,7 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import getTimerange from '../../helpers/get_timerange'; -export default function dateHistogram(req, panel, annotation, capabilities) { +export default function dateHistogram(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const timeField = annotation.time_field; const { bucketSize, intervalString } = getBucketSize(req, 'auto', capabilities); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js index 7c28b84de22eb4..a1d16198112fa6 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js @@ -28,6 +28,9 @@ describe('dateHistogram(req, panel, series)', () => { let series; let req; let capabilities; + let config; + let indexPatternObject; + beforeEach(() => { req = { payload: { @@ -44,18 +47,23 @@ describe('dateHistogram(req, panel, series)', () => { interval: '10s' }; series = { id: 'test' }; + config = { + allowLeadingWildcards: true, + queryStringOptions: {}, + }; + indexPatternObject = {}; capabilities = new DefaultSearchCapabilities(req, true); }); it('calls next when finished', () => { const next = sinon.spy(); - dateHistogram(req, panel, series, capabilities)(next)({}); + dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(next.calledOnce).to.equal(true); }); it('returns valid date histogram', () => { const next = doc => doc; - const doc = dateHistogram(req, panel, series, capabilities)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -87,7 +95,7 @@ describe('dateHistogram(req, panel, series)', () => { it('returns valid date histogram (offset by 1h)', () => { series.offset_time = '1h'; const next = doc => doc; - const doc = dateHistogram(req, panel, series, capabilities)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -122,7 +130,7 @@ describe('dateHistogram(req, panel, series)', () => { series.series_time_field = 'timestamp'; series.series_interval = '20s'; const next = doc => doc; - const doc = dateHistogram(req, panel, series, capabilities)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js index 5dd613be3e8057..6c4934ebc2c157 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js @@ -21,7 +21,7 @@ import getBucketSize from '../../helpers/get_bucket_size'; import offsetTime from '../../offset_time'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; import { set } from 'lodash'; -export default function dateHistogram(req, panel, series, capabilities) { +export default function dateHistogram(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { timeField, interval } = getIntervalAndTimefield(panel, series); const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js index 764919a46b8e55..19aad8f37d8259 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js @@ -21,7 +21,7 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function metricBuckets(req, panel, series, capabilities) { +export default function metricBuckets(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { interval diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js index 03d61508e61bf6..351ff2bf27f629 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js @@ -21,7 +21,7 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function siblingBuckets(req, panel, series, capabilities) { +export default function siblingBuckets(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { interval diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index 8aa160cf6cfbb8..17bdaf220c6622 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -84,34 +84,24 @@ describe('buildRequestBody(req)', () => { const panel = body.panels[0]; const series = panel.series[0]; const getSearchTimezone = sinon.spy(() => 'UTC'); - const getSearchInterval = sinon.spy(); const getValidTimeInterval = sinon.spy(() => '10s'); - const capabilities = { getSearchTimezone, - getSearchInterval, getValidTimeInterval }; - const config = { + const config = { allowLeadingWildcards: true, queryStringOptions: {}, }; - const doc = buildRequestBody({ payload: body }, panel, series, config); + const indexPatternObject = {}; + const doc = buildRequestBody({ payload: body }, panel, series, config, indexPatternObject, capabilities); expect(getSearchTimezone.calledOnce).to.equal(true); expect(doc).to.eql({ size: 0, query: { bool: { + filter: [], must: [ - { - range: { - '@timestamp': { - gte: 1485463055881, - lte: 1485463955881, - format: 'epoch_millis' - } - } - }, { bool: { must: [ @@ -124,41 +114,52 @@ describe('buildRequestBody(req)', () => { ], must_not: [] } - } - ] + }, + { + range: { + '@timestamp': { + gte: 1485463055881, + lte: 1485463955881, + format: 'epoch_millis' + } + } + }, + ], + must_not: [], + should: [], } }, - 'aggs': { - 'timeseries': { - 'aggs': { + aggs: { + timeseries: { + aggs: { 'c9b5f9c1-e403-11e6-be91-6f7688e9fac7': { - 'bucket_script': { - 'buckets_path': { - 'count': '_count' + bucket_script: { + buckets_path: { + count: '_count' }, - 'gap_policy': 'skip', - 'script': { - 'lang': 'expression', - 'source': 'count * 1' + gap_policy: 'skip', + script: { + lang: 'expression', + source: 'count * 1' } } } }, - 'date_histogram': { - 'extended_bounds': { - 'max': 1485463955881, - 'min': 1485463055881 + date_histogram: { + extended_bounds: { + max: 1485463955881, + min: 1485463055881 }, - 'field': '@timestamp', - 'interval': '10s', - 'min_doc_count': 0, - 'time_zone': 'UTC' + field: '@timestamp', + interval: '10s', + min_doc_count: 0, + time_zone: 'UTC' }, - 'meta': { - 'bucketSize': 10, - 'intervalString': '10s', - 'seriesId': 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7', - 'timeField': '@timestamp' + meta: { + bucketSize: 10, + intervalString: '10s', + seriesId: 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7', + timeField: '@timestamp' } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js index 7f17e60e235c55..57f49b923172a8 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js @@ -20,7 +20,6 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/series'; -//todo: req, panel, series, esQueryConfig, indexPatter, capabilities function buildRequestBody(...args) { const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index 8f7cf6e3ac8b4f..20980d81960ae3 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -18,23 +18,24 @@ */ import buildRequestBody from './build_request_body'; import getEsShardTimeout from '../helpers/get_es_shard_timeout'; +import { getIndexPatternObject } from '../helpers/get_index_pattern'; -export default (req, panel, series, esQueryConfig, capabilities) => { +export default async (req, panel, series, esQueryConfig, capabilities) => { const bodies = []; - const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; - const indexPatternObject = await getIndexPatternObject(req, indexPatternString); + const indexPattern = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; + const indexPatternObject = await getIndexPatternObject(req, indexPattern); const timeout = getEsShardTimeout(req); if (capabilities.batchRequestsSupport) { bodies.push({ - index: indexPatternString, + index: indexPattern, ignoreUnavailable: true, }); } bodies.push({ ...buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject, capabilities), - timeout + timeout, }); return bodies; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js index 7e509a5ebf2e5e..349863d9cb6c02 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js @@ -20,8 +20,8 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/table'; -function buildRequestBody(req, panel, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, esQueryConfig, indexPattern); +function buildRequestBody(...args) { + const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } From 712fe25dd4bccc7c914354ccbc7d5ef6ce2026a9 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Thu, 31 Jan 2019 16:06:48 +0300 Subject: [PATCH 44/58] Add table view support --- .../searh_requests/single_search_request.js | 5 +- .../server/lib/vis_data/get_table_data.js | 2 +- .../server/lib/vis_data/helpers/get_splits.js | 4 +- .../request_processors/table/filter_ratios.js | 4 +- .../request_processors/table/index.js | 4 +- .../table/normalize_query.js | 48 +++++++++++++++++++ .../lib/vis_data/table/process_bucket.js | 13 ++++- 7 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js index 95fc0d544abf67..2203c58dec46b3 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -23,11 +23,12 @@ const SEARCH_METHOD = 'search'; export default class MultiSearchRequest extends AbstractSearchRequest { async search(options) { const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); - - return await this.callWithRequest(this.req, SEARCH_METHOD, { + const resp = await this.callWithRequest(this.req, SEARCH_METHOD, { ...options, index: this.indexPattern, ignore_throttled: !includeFrozen, }); + + return [resp]; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index 692d0df75199c7..15e1cc9bc599e8 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -34,7 +34,7 @@ export async function getTableData(req, panel) { const body = buildRequestBody(req, panel, esQueryConfig, indexPatternObject, capabilities); try { - const resp = await searchRequest.search({ body }); + const [resp] = await searchRequest.search({ body }); const buckets = get(resp, 'aggregations.pivot.buckets', []); return { type: 'table', series: buckets.map(processBucket(panel)) }; } catch (err) { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js index bdcca1f69a4a20..75d1b4d565fe51 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js @@ -35,8 +35,8 @@ export default function getSplits(resp, panel, series, meta) { const color = new Color(series.color); const metric = getLastMetric(series); - if (_.has(resp, `aggregations.${series.id}.buckets`)) { - const buckets = _.get(resp, `aggregations.${series.id}.buckets`); + const buckets = _.get(resp, `aggregations.${series.id}.buckets`); + if (buckets) { if (Array.isArray(buckets)) { const size = buckets.length; const colors = getSplitColors(series.color, size, series.split_color_mode); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js index dcafbe39cd153f..ef4bf22e14c0fc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js @@ -23,7 +23,7 @@ import bucketTransform from '../../helpers/bucket_transform'; import _ from 'lodash'; import { calculateAggRoot } from './calculate_agg_root'; export default function ratios(req, panel) { - return () => doc => { + return next => doc => { panel.series.forEach(column => { const aggRoot = calculateAggRoot(doc, column); if (column.metrics.some(filter)) { @@ -63,6 +63,6 @@ export default function ratios(req, panel) { }); } }); - return doc; + return next(doc); }; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js index f2359e5bfffde4..0e160e8333c5ce 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js @@ -25,6 +25,7 @@ import dateHistogram from './date_histogram'; import metricBuckets from './metric_buckets'; import siblingBuckets from './sibling_buckets'; import filterRatios from './filter_ratios'; +import normalizeQuery from './normalize_query'; export default [ query, @@ -34,5 +35,6 @@ export default [ dateHistogram, metricBuckets, siblingBuckets, - filterRatios + filterRatios, + normalizeQuery ]; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js new file mode 100644 index 00000000000000..2c561988b615ce --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +const { set, get, isEmpty, forEach } = require('lodash'); + +const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); + +export default function normalizeQuery() { + return () => doc => { + const series = get(doc, 'aggs.pivot.aggs'); + const normalizedSeries = {}; + + forEach(series, (value, seriesId) => { + const filter = get(value, `filter`); + + if (isEmptyFilter(filter)) { + const agg = get(value, 'aggs.timeseries'); + const meta = { + ...get(value, 'meta'), + seriesId + }; + set(normalizedSeries, `${seriesId}`, agg); + set(normalizedSeries, `${seriesId}.meta`, meta); + } else { + set(normalizedSeries, `${seriesId}`, value); + } + }); + + set(doc, 'aggs.pivot.aggs', normalizedSeries); + + return doc; + }; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js index a42b0f4eefecc5..fcbae1518924eb 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js @@ -21,10 +21,21 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../response_processors/table'; import getLastValue from '../../../../common/get_last_value'; import regression from 'regression'; -import { first, get } from 'lodash'; +import { first, get, set } from 'lodash'; export default function processBucket(panel) { return bucket => { const series = panel.series.map(series => { + const timeseries = get(bucket, `${series.id}.timeseries`); + const buckets = get(bucket, `${series.id}.buckets`); + + if (!timeseries && buckets) { + const meta = get(bucket, `${series.id}.meta`); + const timeseries = { + buckets: get(bucket, `${series.id}.buckets`) + }; + set(bucket, series.id, { meta, timeseries }); + } + const processor = buildProcessorFunction(processors, bucket, panel, series); const result = first(processor([])); if (!result) return null; From be2ba07b9a87b7b189c65e28547684a6fe06d3cb Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 31 Jan 2019 16:34:10 +0300 Subject: [PATCH 45/58] fix broken build --- .../visualize/loader/pipeline_helpers/build_pipeline.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts index 62d2212587a1e0..7297c1c1359be5 100644 --- a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts +++ b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts @@ -201,11 +201,11 @@ export const buildPipelineVisFunction: BuildPipelineVisFunction = { input_control_vis: visState => { return `input_control_vis ${prepareJson('visConfig', visState.params)}`; }, - metrics: (visState, schemas, uiState) => { - const params = prepareJson('params', visState.params); - const uiState = prepareJson('uiState', uiState); + metrics: (visState, schemas, uiState = {}) => { + const paramsJson = prepareJson('params', visState.params); + const uiStateJson = prepareJson('uiState', uiState); - return `tsvb ${params} ${uiState}`; + return `tsvb ${paramsJson} ${uiStateJson}`; }, timelion: visState => { const expression = prepareString('expression', visState.params.expression); From 2db4672ea11857e277788de986a0d24296693d94 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 1 Feb 2019 11:32:33 +0300 Subject: [PATCH 46/58] fix broken build --- .../pipeline_helpers/__snapshots__/build_pipeline.test.js.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap b/src/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap index de3a947a784ed8..b6eedb6f1e4536 100644 --- a/src/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap +++ b/src/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap @@ -8,7 +8,7 @@ exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunct exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metric function without buckets 1`] = `"kibana_metric visConfig='{\\"metric\\":{\\"metrics\\":[0,1]}}' "`; -exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metrics/tsvb function 1`] = `"tsvb params='{\\"foo\\":\\"bar\\"}' "`; +exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metrics/tsvb function 1`] = `"tsvb params='{\\"foo\\":\\"bar\\"}' uiState='{}' "`; exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles pie function 1`] = `"kibana_pie visConfig='{\\"dimensions\\":{\\"metric\\":0,\\"buckets\\":[1,2]}}' "`; From 68e666ddf69dc27326c5c446cc8740a2b147f56e Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 1 Feb 2019 13:45:41 +0300 Subject: [PATCH 47/58] [Rollup] [Phase 1] - write new tests (rollup_search_request, rollup_search_strategy) --- .../register_rollup_search_strategy.test.js | 52 +++++++++++++++++++ .../rollup_search_request.test.js | 50 ++++++++++++++++++ .../rollup_search_strategy.js | 1 - 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js new file mode 100644 index 00000000000000..488cf102b7dd8a --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js @@ -0,0 +1,52 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import registerRollupSearchStrategy from './register_rollup_search_strategy'; + +describe('Register Rollup Search Strategy', () => { + let kbnServer; + let metrics; + + beforeEach(() => { + const afterPluginsInit = jest.fn((callback) => callback()); + + kbnServer = { + afterPluginsInit, + }; + + metrics = { + addSearchStrategy: jest.fn().mockName('addSearchStrategy'), + AbstractSearchRequest: jest.fn().mockName('AbstractSearchRequest'), + AbstractSearchStrategy: jest.fn().mockName('AbstractSearchStrategy'), + DefaultSearchCapabilities: jest.fn().mockName('DefaultSearchCapabilities'), + }; + }); + + test('should run initialization on "afterPluginsInit" hook', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: {}, + }); + + expect(kbnServer.afterPluginsInit).toHaveBeenCalled(); + }); + + test('should run initialization if metrics plugin available', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: { + metrics, + }, + }); + + expect(metrics.addSearchStrategy).toHaveBeenCalled(); + }); + + test('should not run initialization if metrics plugin unavailable', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: {}, + }); + + expect(metrics.addSearchStrategy).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js new file mode 100644 index 00000000000000..ab6f8794088064 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js @@ -0,0 +1,50 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import getRollupSearchRequest from './rollup_search_request'; + +class AbstractSearchRequest { + indexPattern = 'indexPattern'; + callWithRequest = jest.fn(({ body }) => Promise.resolve(body)); +} + +describe('Rollup search request', () => { + let RollupSearchRequest; + + beforeEach(() => { + RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); + }); + + test('should create instance of RollupSearchRequest', () => { + const rollupSearchRequest = new RollupSearchRequest(); + + expect(rollupSearchRequest.indexPattern).toBeDefined(); + expect(rollupSearchRequest.callWithRequest).toBeDefined(); + expect(rollupSearchRequest.search).toBeDefined(); + }); + + test('should send one request for single search', async () => { + const rollupSearchRequest = new RollupSearchRequest(); + const body = 'body'; + + await rollupSearchRequest.search({ body }); + + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledTimes(1); + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledWith('rollup.search', { + body, + index: 'indexPattern', + rest_total_hits_as_int: true, + }); + }); + + test('should send multiple request for multi search', async () => { + const rollupSearchRequest = new RollupSearchRequest(); + const body = ['firstRequestBody', 'secondRequestBody']; + + await rollupSearchRequest.search({ body }); + + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledTimes(body.length); + }); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 5dcec141f2cf9f..6f1c627df393c7 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -57,7 +57,6 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil async getFieldsForWildcard(req, indexPattern, { fieldsCapabilities, rollupIndex }) { const fields = await super.getFieldsForWildcard(req, indexPattern); - const fieldsFromFieldCapsApi = indexBy(fields, 'name'); const rollupIndexCapabilities = fieldsCapabilities[rollupIndex].aggs; From 751d0c06bd18365a485573a51b90f440d1323b32 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 1 Feb 2019 17:19:03 +0300 Subject: [PATCH 48/58] [Rollup] [Phase 1] - write tests for rollup_search_capabilities --- .../rollup_search_capabilities.test.js | 103 ++++++++++++++++++ .../rollup_search_request.test.js | 3 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js new file mode 100644 index 00000000000000..30a60a9802cf5a --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -0,0 +1,103 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import getRollupSearchCapabilities from './rollup_search_capabilities'; + +class DefaultSearchCapabilities { + constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { + this.fieldsCapabilities = fieldsCapabilities; + this.validateTimeIntervalRules = []; + } + + get defaultTimeIntervalInSeconds() { + return this.getIntervalInSeconds(this.defaultTimeInterval); + } + + getIntervalInSeconds(intervalString) { + return Number.parseInt(intervalString); + } + + isTimeIntervalValid(interval) { + return this.validateTimeIntervalRules + .every(validationRule => validationRule(this.getIntervalInSeconds(interval), this.defaultTimeIntervalInSeconds)); + } +} + +describe('Rollup Search Capabilities', () => { + const testTimeZone = 'time_zone'; + const testInterval = '10s'; + const rollupIndex = 'rollupIndex'; + const batchRequestsSupport = true; + const request = {}; + + let RollupSearchCapabilities; + let fieldsCapabilities; + let rollupSearchCaps; + + beforeEach(() => { + RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); + fieldsCapabilities = { + [rollupIndex]: { + aggs: { + date_histogram: { + histogram_field: { + time_zone: testTimeZone, + interval: testInterval, + }, + }, + }, + }, + }; + + rollupSearchCaps = new RollupSearchCapabilities(request, batchRequestsSupport, fieldsCapabilities, rollupIndex); + }); + + test('should create instance of RollupSearchRequest', () => { + expect(rollupSearchCaps).toBeInstanceOf(DefaultSearchCapabilities); + expect(rollupSearchCaps.isTimeIntervalValid).toBeDefined(); + expect(rollupSearchCaps.fieldsCapabilities).toBe(fieldsCapabilities); + expect(rollupSearchCaps.rollupIndex).toBe(rollupIndex); + }); + + test('should return the "timezone" for the rollup request', () => { + expect(rollupSearchCaps.fixedTimeZone).toBe(testTimeZone); + }); + + test('should return the default "interval" for the rollup request', () => { + expect(rollupSearchCaps.defaultTimeInterval).toBe(testInterval); + }); + + describe('intervalMultiple', () => { + let intervalMultiple; + + beforeEach(() => { + [intervalMultiple] = rollupSearchCaps.validateTimeIntervalRules; + }); + + test('should add intervalMultiple into validation rules', () => { + expect(intervalMultiple).toBeDefined(); + }); + + test('should return true for multiple intervals', () => { + expect(intervalMultiple(6, 3)).toBeTruthy(); + expect(intervalMultiple(10, 5)).toBeTruthy(); + }); + + test('should return false for not multiple intervals', () => { + expect(intervalMultiple(7, 3)).toBeFalsy(); + expect(intervalMultiple(21, 10)).toBeFalsy(); + }); + }); + + describe('getValidTimeInterval', () => { + test('interval should be greater than default interval', () => { + expect(rollupSearchCaps.getValidTimeInterval('1s')).toBe(testInterval); + }); + + test('should round interval', () => { + expect(rollupSearchCaps.getValidTimeInterval('11s')).toBe('20s'); + }); + }); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js index ab6f8794088064..71910e591f04b3 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js @@ -20,9 +20,10 @@ describe('Rollup search request', () => { test('should create instance of RollupSearchRequest', () => { const rollupSearchRequest = new RollupSearchRequest(); + expect(rollupSearchRequest).toBeInstanceOf(AbstractSearchRequest); + expect(rollupSearchRequest.search).toBeDefined(); expect(rollupSearchRequest.indexPattern).toBeDefined(); expect(rollupSearchRequest.callWithRequest).toBeDefined(); - expect(rollupSearchRequest.search).toBeDefined(); }); test('should send one request for single search', async () => { From 1bd307640b6cf115850a192f1a3843d0fea2503a Mon Sep 17 00:00:00 2001 From: sulemanof Date: Fri, 1 Feb 2019 18:23:55 +0300 Subject: [PATCH 49/58] Add test on default_search_capabilities, abstract_search_strategy, search_strategies_register --- .../default_search_capabilities.test.js | 98 +++++++++++++++++++ .../search_strategies_register.js | 2 +- .../search_strategies_register.test.js | 94 ++++++++++++++++++ .../abstract_search_strategy.test.js | 82 ++++++++++++++++ 4 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js new file mode 100644 index 00000000000000..e28c7c4d9004d9 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import DefaultSearchCapabilities from './default_search_capabilities'; + +describe('DefaultSearchCapabilities', () => { + let defaultSearchCapabilities; + let req; + let batchRequestsSupport; + + beforeEach(() => { + req = {}; + batchRequestsSupport = true; + defaultSearchCapabilities = new DefaultSearchCapabilities(req, batchRequestsSupport); + }); + + test('should init default search capabilities', () => { + expect(defaultSearchCapabilities.request).toBe(req); + expect(defaultSearchCapabilities.batchRequestsSupport).toBe(batchRequestsSupport); + expect(defaultSearchCapabilities.fieldsCapabilities).toEqual({}); + expect(defaultSearchCapabilities.validateTimeIntervalRules).toEqual([]); + }); + + test('should return fixedTimeZone', () => { + expect(defaultSearchCapabilities.fixedTimeZone).toBe(null); + }); + + test('should return defaultTimeInterval', () => { + expect(defaultSearchCapabilities.defaultTimeInterval).toBe(null); + }); + + test('should return defaultTimeIntervalInSeconds', () => { + defaultSearchCapabilities.getIntervalInSeconds = jest.fn(() => '20m'); + + expect(defaultSearchCapabilities.defaultTimeIntervalInSeconds).toEqual('20m'); + expect(defaultSearchCapabilities.getIntervalInSeconds) + .toHaveBeenCalledWith(defaultSearchCapabilities.defaultTimeInterval); + }); + + test('should return Search Timezone', () => { + defaultSearchCapabilities.request = { + payload: { + timerange: { + timezone: 'UTC' + } + } + }; + + expect(defaultSearchCapabilities.getSearchTimezone()).toEqual('UTC'); + }); + + test('should return interval in seconds', () => { + expect(defaultSearchCapabilities.getIntervalInSeconds()).toEqual(0); + expect(defaultSearchCapabilities.getIntervalInSeconds('20m')).toEqual(1200); + expect(defaultSearchCapabilities.getIntervalInSeconds('1h')).toEqual(3600); + }); + + test('should check if a time interval is valid', () => { + defaultSearchCapabilities.validateTimeIntervalRules.push( + () => true + ); + + expect(defaultSearchCapabilities.isTimeIntervalValid()).toBe(true); + expect(defaultSearchCapabilities.isTimeIntervalValid('20m')).toBe(true); + expect(defaultSearchCapabilities.isTimeIntervalValid('1h')).toBe(true); + + defaultSearchCapabilities.validateTimeIntervalRules.push( + () => false + ); + + expect(defaultSearchCapabilities.isTimeIntervalValid('20m')).toBe(false); + }); + + test('should return a valid time interval', () => { + defaultSearchCapabilities.isTimeIntervalValid = jest.fn(() => true); + + expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe('20m'); + + defaultSearchCapabilities.isTimeIntervalValid = jest.fn(() => false); + + expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe(null); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index e8e3b8ee3119c1..f577bb446b77b3 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -27,7 +27,7 @@ const addStrategy = searchStrategy => { if (searchStrategy instanceof AbstractSearchStrategy) { strategies.unshift(searchStrategy); } - return this; + return strategies; }; export default class SearchStrategiesRegister { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js new file mode 100644 index 00000000000000..891319ca696393 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import SearchStrategiesRegister from './search_strategies_register'; +import AbstractSearchStrategy from './strategies/abstract_search_strategy'; +import DefaultSearchStrategy from './strategies/default_search_strategy'; +import AbstractSearchRequest from './searh_requests/abstract_request'; +import DefaultSearchCapabilities from './default_search_capabilities'; + +class MockSearchStrategy extends AbstractSearchStrategy { + checkForViability() { + return { + isViable: true, + capabilities: {} + }; + } +} + +describe('SearchStrategiesRegister', () => { + let server; + let strategies; + let anotherSearchStrategy; + + beforeAll(() => { + server = { + expose: jest.fn((strategy, func) => { + server[strategy] = func; + }) + }; + strategies = [ + ['AbstractSearchStrategy', AbstractSearchStrategy], + ['AbstractSearchRequest', AbstractSearchRequest], + ['DefaultSearchCapabilities', DefaultSearchCapabilities], + ['addSearchStrategy', expect.any(Function)] + ]; + + SearchStrategiesRegister.init(server); + }); + + test('should init strategies register', () => { + expect(server.expose.mock.calls).toEqual(strategies); + expect(server.addSearchStrategy()[0] instanceof DefaultSearchStrategy).toBe(true); + }); + + test('should not add a strategy if it is not an instance of AbstractSearchStrategy', () => { + const addedStrategies = server.addSearchStrategy({}); + + expect(addedStrategies.length).toEqual(1); + expect(addedStrategies[0] instanceof DefaultSearchStrategy).toBe(true); + }); + + test('should return a DefaultSearchStrategy instance', async () => { + const req = {}; + const indexPattern = '*'; + + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + + expect(searchStrategy instanceof DefaultSearchStrategy).toBe(true); + expect(capabilities instanceof DefaultSearchCapabilities).toBe(true); + }); + + test('should add a strategy if it is an instance of AbstractSearchStrategy', () => { + anotherSearchStrategy = new MockSearchStrategy(); + const addedStrategies = server.addSearchStrategy(anotherSearchStrategy); + + expect(addedStrategies.length).toEqual(2); + expect(addedStrategies[0] instanceof AbstractSearchStrategy).toBe(true); + }); + + test('should return a MockSearchStrategy instance', async () => { + const req = {}; + const indexPattern = '*'; + + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + + expect(searchStrategy instanceof AbstractSearchStrategy).toBe(true); + expect(capabilities).toEqual({}); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js new file mode 100644 index 00000000000000..3b04ae69f4014c --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchStrategy from './abstract_search_strategy'; + +class SearchRequest { + constructor(req, callWithRequest, indexPattern) { + this.req = req; + this.callWithRequest = callWithRequest; + this.indexPattern = indexPattern; + } +} + +describe('AbstractSearchStrategy', () => { + let abstractSearchStrategy; + let server; + let callWithRequestFactory; + let req; + let mockedFields; + let indexPattern; + + beforeEach(() => { + server = {}; + callWithRequestFactory = jest.fn().mockReturnValue('callWithRequest'); + mockedFields = {}; + indexPattern = '*'; + req = { + pre: { + indexPatternsService: { + getFieldsForWildcard: jest.fn().mockReturnValue(mockedFields) + } + } + }; + + abstractSearchStrategy = new AbstractSearchStrategy(server, callWithRequestFactory, SearchRequest); + }); + + test('should init an AbstractSearchStrategy instance', () => { + expect(abstractSearchStrategy.getCallWithRequestInstance).toBeDefined(); + expect(abstractSearchStrategy.getSearchRequest).toBeDefined(); + expect(abstractSearchStrategy.getFieldsForWildcard).toBeDefined(); + }); + + test('should return fields for wildcard', async () => { + const fields = await abstractSearchStrategy.getFieldsForWildcard(req, indexPattern); + + expect(fields).toBe(mockedFields); + expect(req.pre.indexPatternsService.getFieldsForWildcard).toHaveBeenCalledWith({ + pattern: indexPattern, + }); + }); + + test('should invoke callWithRequestFactory with req param passed', () => { + abstractSearchStrategy.getCallWithRequestInstance(req); + + expect(callWithRequestFactory).toHaveBeenCalledWith(server, req); + }); + + test('should return a search request', () => { + const searchRequest = abstractSearchStrategy.getSearchRequest(req, indexPattern); + + expect(searchRequest instanceof SearchRequest).toBe(true); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.callWithRequest).toBe('callWithRequest'); + expect(searchRequest.req).toBe(req); + }); +}); From 8d4d112a1d45b65dd05a0b9ff2107432853e94b0 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Mon, 4 Feb 2019 14:01:28 +0300 Subject: [PATCH 50/58] Add test cases for search_requests folder --- .../searh_requests/abstract_request.test.js | 49 ++++++++++++ .../multi_search_request.test.js | 61 +++++++++++++++ .../searh_requests/search_request.test.js | 78 +++++++++++++++++++ .../searh_requests/single_search_request.js | 2 +- .../single_search_request.test.js | 61 +++++++++++++++ .../default_search_strategy.test.js | 70 +++++++++++++++++ 6 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js new file mode 100644 index 00000000000000..4f8b10a2ef1248 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import AbstractSearchRequest from './abstract_request'; + +describe('AbstractSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + + beforeEach(() => { + req = {}; + callWithRequest = jest.fn(); + indexPattern = 'indexPattern'; + searchRequest = new AbstractSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an AbstractSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should throw an error trying to search', () => { + try { + searchRequest.search(); + } catch (error) { + expect(error instanceof Error).toBe(true); + expect(error.message).toEqual('AbstractSearchRequest: search method should be defined'); + } + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js new file mode 100644 index 00000000000000..ef9c5fc32adf15 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import MultiSearchRequest from './multi_search_request'; + +describe('MultiSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({ responses: [] }); + indexPattern = 'indexPattern'; + searchRequest = new MultiSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an MultiSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should get the response from elastic msearch', async () => { + const options = {}; + + const responses = await searchRequest.search(options); + + expect(responses).toEqual([]); + expect(req.getUiSettingsService).toHaveBeenCalled(); + expect(getServiceMock).toHaveBeenCalledWith('search:includeFrozen'); + expect(callWithRequest).toHaveBeenCalledWith(req, 'msearch', { + ...options, + rest_total_hits_as_int: true, + ignore_throttled: !includeFrozen, + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js new file mode 100644 index 00000000000000..e0999e9c91b71c --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import SearchRequest from './search_request'; +import MultiSearchRequest from './multi_search_request'; +import SingleSearchRequest from './single_search_request'; + +describe('SearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({ responses: [] }); + indexPattern = 'indexPattern'; + searchRequest = new SearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an AbstractSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should return search value', async () => { + const concreteSearchRequest = { + search: jest.fn().mockReturnValue('concreteSearchRequest') + }; + const options = {}; + searchRequest.getSearchRequestType = jest.fn().mockReturnValue(concreteSearchRequest); + + const result = await searchRequest.search(options); + + expect(result).toBe('concreteSearchRequest'); + }); + + test('should return a MultiSearchRequest if options has body as an array', () => { + const options = { + body: [] + }; + + const result = searchRequest.getSearchRequestType(options); + + expect(result instanceof MultiSearchRequest).toBe(true); + }); + + test('should return a SingleSearchRequest if options has body', () => { + const options = {}; + + const result = searchRequest.getSearchRequestType(options); + + expect(result instanceof SingleSearchRequest).toBe(true); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js index 2203c58dec46b3..189b226638a563 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -20,7 +20,7 @@ import AbstractSearchRequest from './abstract_request'; const SEARCH_METHOD = 'search'; -export default class MultiSearchRequest extends AbstractSearchRequest { +export default class SingleSearchRequest extends AbstractSearchRequest { async search(options) { const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); const resp = await this.callWithRequest(this.req, SEARCH_METHOD, { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js new file mode 100644 index 00000000000000..c10409f1acd4bd --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import SingleSearchRequest from './single_search_request'; + +describe('SingleSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({}); + indexPattern = 'indexPattern'; + searchRequest = new SingleSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an SingleSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should get the response from elastic search', async () => { + const options = {}; + + const responses = await searchRequest.search(options); + + expect(responses).toEqual([{}]); + expect(req.getUiSettingsService).toHaveBeenCalled(); + expect(getServiceMock).toHaveBeenCalledWith('search:includeFrozen'); + expect(callWithRequest).toHaveBeenCalledWith(req, 'search', { + ...options, + index: indexPattern, + ignore_throttled: !includeFrozen, + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js new file mode 100644 index 00000000000000..83930105b409dc --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import DefaultSearchStrategy from './default_search_strategy'; + +describe('DefaultSearchStrategy', () => { + let defaultSearchStrategy; + let server; + let callWithRequest; + let req; + + beforeEach(() => { + server = {}; + callWithRequest = jest.fn(); + req = { + server: { + plugins: { + elasticsearch: { + getCluster: jest.fn().mockReturnValue({ + callWithRequest + }) + } + } + } + }; + defaultSearchStrategy = new DefaultSearchStrategy(server); + }); + + test('should init an DefaultSearchStrategy instance', () => { + expect(defaultSearchStrategy.name).toBe('default'); + expect(defaultSearchStrategy.checkForViability).toBeDefined(); + expect(defaultSearchStrategy.getCallWithRequestInstance).toBeDefined(); + expect(defaultSearchStrategy.getSearchRequest).toBeDefined(); + expect(defaultSearchStrategy.getFieldsForWildcard).toBeDefined(); + }); + + test('should invoke callWithRequestFactory with passed params', () => { + const value = defaultSearchStrategy.getCallWithRequestInstance(req); + + expect(value).toBe(callWithRequest); + expect(req.server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith('data'); + }); + + test('should check a strategy for viability', () => { + const value = defaultSearchStrategy.checkForViability(req); + + expect(value.isViable).toBe(true); + expect(value.capabilities).toEqual({ + request: req, + batchRequestsSupport: true, + fieldsCapabilities: {}, + validateTimeIntervalRules: [] + }); + }); +}); From f3312eb58547106beb57f6b06cdd0aa7f90bb233 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 4 Feb 2019 16:19:43 +0300 Subject: [PATCH 51/58] [Rollup] [Phase 1] - write tests for rollup_search_strategy --- .../rollup_search_strategy.test.js | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js new file mode 100644 index 00000000000000..0cd2354d96a4cd --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js @@ -0,0 +1,153 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import getRollupSearchStrategy from './rollup_search_strategy'; + +describe('Rollup Search Strategy', () => { + let RollupSearchStrategy; + let RollupSearchRequest; + let RollupSearchCapabilities; + let callWithRequest; + let rollupResolvedData; + + const server = 'server'; + const request = 'request'; + const indexPattern = 'indexPattern'; + + beforeEach(() => { + class AbstractSearchStrategy { + getCallWithRequestInstance = jest.fn(() => callWithRequest); + + getFieldsForWildcard() { + return [ + { + name: 'day_of_week.terms.value', + type: 'object', + searchable: false, + aggregatable: false, + }, + ]; + } + } + + RollupSearchRequest = jest.fn(); + RollupSearchCapabilities = jest.fn(() => 'capabilities'); + callWithRequest = jest.fn().mockImplementation(() => rollupResolvedData); + + RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); + }); + + test('should create instance of RollupSearchRequest', () => { + const rollupSearchStrategy = new RollupSearchStrategy(server); + + expect(rollupSearchStrategy.name).toBe('rollup'); + }); + + describe('checkForViability', () => { + let rollupSearchStrategy; + const rollupIndex = 'rollupIndex'; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + rollupSearchStrategy.getRollupData = jest.fn(() => ({ + [rollupIndex]: { + rollup_jobs: [{ + job_id: 'test', + rollup_index: rollupIndex, + index_pattern: 'kibana*', + fields: { + order_date: [{ + agg: 'date_histogram', + delay: '1m', + interval: '1m', + time_zone: 'UTC', + }], + day_of_week: [{ + agg: 'terms', + }], + }, + }], + }, + })); + }); + + test('isViable should be false for invalid index', async () => { + const result = await rollupSearchStrategy.checkForViability(request, null); + + expect(result).toEqual({ + isViable: false, + capabilities: null, + }); + }); + + test('should get RollupSearchCapabilities for valid rollup index ', async () => { + await rollupSearchStrategy.checkForViability(request, rollupIndex); + + expect(RollupSearchCapabilities).toHaveBeenCalled(); + }); + }); + + describe('getRollupData', () => { + let rollupSearchStrategy; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + }); + + test('should return rollup data', async () => { + rollupResolvedData = Promise.resolve('data'); + + const rollupData = await rollupSearchStrategy.getRollupData(request, indexPattern); + + expect(callWithRequest).toHaveBeenCalledWith('rollup.rollupIndexCapabilities', { indexPattern }); + expect(rollupSearchStrategy.getCallWithRequestInstance).toHaveBeenCalledWith(request); + expect(rollupData).toBe('data'); + }); + + test('should return empty object in case of exception', async () => { + rollupResolvedData = Promise.reject('data'); + + const rollupData = await rollupSearchStrategy.getRollupData(request, indexPattern); + + expect(rollupData).toEqual({}); + }); + }); + + describe('getFieldsForWildcard', () => { + let rollupSearchStrategy; + let fieldsCapabilities; + + const rollupIndex = 'rollupIndex'; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + fieldsCapabilities = { + [rollupIndex]: { + aggs: { + terms: { + day_of_week: { agg: 'terms' }, + }, + }, + }, + }; + }); + + test('should return fields for wildcard', async () => { + const fields = await rollupSearchStrategy.getFieldsForWildcard(request, indexPattern, + { fieldsCapabilities, rollupIndex }, + ); + + expect(fields).toEqual([ + { + aggregatable: true, + name: 'day_of_week', + readFromDocValues: true, + searchable: true, + type: 'object', + }, + ]); + }); + }); +}); From 97f5315303a38843d4750165f93e6ee73a89a2a7 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Thu, 7 Feb 2019 12:42:10 +0300 Subject: [PATCH 52/58] FIx broken build --- .../loader/pipeline_helpers/build_pipeline.ts | 792 +++++++++--------- 1 file changed, 396 insertions(+), 396 deletions(-) diff --git a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts index ded23c99b3ce83..63980298790411 100644 --- a/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts +++ b/src/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts @@ -1,396 +1,396 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -// @ts-ignore -import { setBounds } from 'ui/agg_types/buckets/date_histogram'; -import { SearchSource } from 'ui/courier'; -import { AggConfig, Vis, VisState } from 'ui/vis'; - -interface SchemaFormat { - id: string; - params?: any; -} - -interface SchemaConfig { - accessor: number; - format: SchemaFormat | {}; - params: any; - aggType: string; -} - -interface Schemas { - metric: SchemaConfig[]; - bucket?: any[]; - geo_centroid?: any[]; - group?: any[]; - params?: any[]; - radius?: any[]; - segment?: any[]; - split_column?: any[]; - split_row?: any[]; - width?: any[]; - // catch all for schema name - [key: string]: any[] | undefined; -} - -type buildVisFunction = (visState: VisState, schemas: Schemas) => string; -type buildVisConfigFunction = (visState: Vis, schemas: Schemas, uiState: any) => VisState; - -interface BuildPipelineVisFunction { - [key: string]: buildVisFunction; -} - -interface BuildVisConfigFunction { - [key: string]: buildVisConfigFunction; -} - -const vislibCharts: string[] = [ - 'area', - 'gauge', - 'goal', - 'heatmap', - 'histogram', - 'horizontal_bar', - 'line', -]; - -export const getSchemas = (vis: Vis, timeRange?: any): Schemas => { - const createFormat = (agg: AggConfig): SchemaFormat => { - const format: SchemaFormat = agg.params.field ? agg.params.field.format.toJSON() : {}; - const formats: any = { - date_range: () => ({ id: 'string' }), - percentile_ranks: () => ({ id: 'percent' }), - count: () => ({ id: 'number' }), - cardinality: () => ({ id: 'number' }), - date_histogram: () => ({ - id: 'date', - params: { - pattern: agg.buckets.getScaledDateFormat(), - }, - }), - terms: () => ({ - id: 'terms', - params: { - id: format.id, - otherBucketLabel: agg.params.otherBucketLabel, - missingBucketLabel: agg.params.missingBucketLabel, - ...format.params, - }, - }), - range: () => ({ - id: 'range', - params: { id: format.id, ...format.params }, - }), - }; - - return formats[agg.type.name] ? formats[agg.type.name]() : format; - }; - - const createSchemaConfig = (accessor: number, agg: AggConfig): SchemaConfig => { - const schema = { - accessor, - format: {}, - params: {}, - aggType: agg.type.name, - }; - - if (agg.type.name === 'date_histogram') { - agg.params.timeRange = timeRange; - setBounds(agg, true); - } - if (agg.type.name === 'geohash_grid') { - schema.params = { - precision: agg.params.precision, - useGeocentroid: agg.params.useGeocentroid, - }; - } - - if ( - [ - 'derivative', - 'moving_avg', - 'serial_diff', - 'cumulative_sum', - 'sum_bucket', - 'avg_bucket', - 'min_bucket', - 'max_bucket', - ].includes(agg.type.name) - ) { - const subAgg = agg.params.customMetric || agg.aggConfigs.byId[agg.params.metricAgg]; - schema.format = createFormat(subAgg); - } else { - schema.format = createFormat(agg); - } - - return schema; - }; - - let cnt = 0; - const schemas: Schemas = { - metric: [], - }; - const responseAggs = vis.aggs.getResponseAggs().filter((agg: AggConfig) => agg.enabled); - const isHierarchical = vis.isHierarchical(); - const metrics = responseAggs.filter((agg: AggConfig) => agg.type.type === 'metrics'); - responseAggs.forEach((agg: AggConfig) => { - if (!agg.enabled) { - cnt++; - return; - } - let skipMetrics = false; - let schemaName = agg.schema ? agg.schema.name || agg.schema : null; - if (typeof schemaName === 'object') { - schemaName = null; - } - if (!schemaName) { - if (agg.type.name === 'geo_centroid') { - schemaName = 'geo_centroid'; - } else { - cnt++; - return; - } - } - if (schemaName === 'split') { - schemaName = `split_${agg.params.row ? 'row' : 'column'}`; - skipMetrics = true; - } - if (!schemas[schemaName]) { - schemas[schemaName] = []; - } - if (!isHierarchical || agg.type.type !== 'metrics') { - schemas[schemaName]!.push(createSchemaConfig(cnt++, agg)); - } - if (isHierarchical && (agg.type.type !== 'metrics' || metrics.length === responseAggs.length)) { - metrics.forEach((metric: any) => { - const schemaConfig = createSchemaConfig(cnt++, metric); - if (!skipMetrics) { - schemas.metric.push(schemaConfig); - } - }); - } - }); - return schemas; -}; - -export const prepareJson = (variable: string, data: object): string => { - return `${variable}='${JSON.stringify(data) - .replace(/\\/g, `\\\\`) - .replace(/'/g, `\\'`)}' `; -}; - -export const prepareString = (variable: string, data: string): string => { - return `${variable}='${data.replace(/\\/g, `\\\\`).replace(/'/g, `\\'`)}' `; -}; - -export const buildPipelineVisFunction: BuildPipelineVisFunction = { - vega: visState => { - return `vega ${prepareString('spec', visState.params.spec)}`; - }, - input_control_vis: visState => { - return `input_control_vis ${prepareJson('visConfig', visState.params)}`; - }, - metrics: (visState, schemas, uiState = {}) => { - const paramsJson = prepareJson('params', visState.params); - const uiStateJson = prepareJson('uiState', uiState); - - return `tsvb ${paramsJson} ${uiStateJson}`; - }, - timelion: visState => { - const expression = prepareString('expression', visState.params.expression); - const interval = prepareString('interval', visState.params.interval); - return `timelion_vis ${expression}${interval}`; - }, - markdown: visState => { - const expression = prepareString('expression', visState.params.markdown); - const visConfig = prepareJson('visConfig', visState.params); - return `kibana_markdown ${expression}${visConfig}`; - }, - table: (visState, schemas) => { - const visConfig = buildVisConfig.table(visState, schemas); - return `kibana_table ${prepareJson('visConfig', visConfig)}`; - }, - metric: (visState, schemas) => { - const visConfig = buildVisConfig.metric(visState, schemas); - return `kibana_metric ${prepareJson('visConfig', visConfig)}`; - }, - tagcloud: (visState, schemas) => { - const visConfig = buildVisConfig.tagcloud(visState, schemas); - return `tagcloud ${prepareJson('visConfig', visConfig)}`; - }, - region_map: (visState, schemas) => { - const visConfig = buildVisConfig.region_map(visState, schemas); - return `regionmap ${prepareJson('visConfig', visConfig)}`; - }, - tile_map: (visState, schemas) => { - const visConfig = buildVisConfig.tile_map(visState, schemas); - return `tilemap ${prepareJson('visConfig', visConfig)}`; - }, - pie: (visState, schemas) => { - const visConfig = buildVisConfig.pie(visState, schemas); - return `kibana_pie ${prepareJson('visConfig', visConfig)}`; - }, -}; - -const buildVisConfig: BuildVisConfigFunction = { - table: (visState, schemas) => { - const visConfig = visState.params; - visConfig.dimensions = { - metrics: schemas.metric, - buckets: schemas.bucket || [], - splitRow: schemas.split_row, - splitColumn: schemas.split_column, - }; - return visConfig; - }, - metric: (visState, schemas) => { - const visConfig = visState.params; - visConfig.metric.metrics = schemas.metric; - if (schemas.group) { - visConfig.metric.bucket = schemas.group[0]; - } - return visConfig; - }, - tagcloud: (visState, schemas) => { - const visConfig = visState.params; - visConfig.metric = schemas.metric[0]; - if (schemas.segment) { - visConfig.bucket = schemas.segment[0]; - } - return visConfig; - }, - region_map: (visState, schemas) => { - const visConfig = visState.params; - visConfig.metric = schemas.metric[0]; - if (schemas.segment) { - visConfig.bucket = schemas.segment[0]; - } - return visConfig; - }, - tile_map: (visState, schemas) => { - const visConfig = visState.params; - visConfig.dimensions = { - metric: schemas.metric[0], - geohash: schemas.segment ? schemas.segment[0] : null, - geocentroid: schemas.geo_centroid ? schemas.geo_centroid[0] : null, - }; - return visConfig; - }, - pie: (visState, schemas) => { - const visConfig = visState.params; - visConfig.dimensions = { - metric: schemas.metric[0], - buckets: schemas.segment, - splitRow: schemas.split_row, - splitColumn: schemas.split_column, - }; - return visConfig; - }, -}; - -export const buildVislibDimensions = (vis: any, timeRange?: any) => { - const schemas = getSchemas(vis, timeRange); - const dimensions = { - x: schemas.segment ? schemas.segment[0] : null, - y: schemas.metric, - z: schemas.radius, - width: schemas.width, - series: schemas.group, - splitRow: schemas.split_row, - splitColumn: schemas.split_column, - }; - if (schemas.segment) { - const xAgg = vis.aggs.getResponseAggs()[dimensions.x.accessor]; - if (xAgg.type.name === 'date_histogram') { - dimensions.x.params.date = true; - dimensions.x.params.interval = xAgg.buckets.getInterval().asMilliseconds(); - dimensions.x.params.format = xAgg.buckets.getScaledDateFormat(); - dimensions.x.params.bounds = xAgg.buckets.getBounds(); - } - } - - return dimensions; -}; - -// If not using the expression pipeline (i.e. visualize_data_loader), we need a mechanism to -// take a Vis object and decorate it with the necessary params (dimensions, bucket, metric, etc) -export const decorateVisObject = (vis: Vis, params: { timeRange?: any }) => { - const schemas = getSchemas(vis, params.timeRange); - let visConfig = vis.params; - if (buildVisConfig[vis.type.name]) { - visConfig = buildVisConfig[vis.type.name](vis, schemas); - vis.params = visConfig; - } else if (vislibCharts.includes(vis.type.name)) { - visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); - } -}; - -export const buildPipeline = ( - vis: Vis, - params: { searchSource: SearchSource; timeRange?: any } -) => { - const { searchSource } = params; - const { indexPattern } = vis; - const query = searchSource.getField('query'); - const filters = searchSource.getField('filter'); - const visState = vis.getCurrentState(); - const uiState = vis.getUiState(); - - // context - let pipeline = `kibana | kibana_context `; - if (query) { - pipeline += prepareJson('query', query); - } - if (filters) { - pipeline += prepareJson('filters', filters); - } - if (vis.savedSearchId) { - pipeline += prepareString('savedSearchId', vis.savedSearchId); - } - pipeline += '| '; - - // request handler - if (vis.type.requestHandler === 'courier') { - pipeline += `esaggs - ${prepareString('index', indexPattern.id)} - metricsAtAllLevels=${vis.isHierarchical()} - partialRows=${vis.params.showPartialRows || vis.type.requiresPartialRows || false} - ${prepareJson('aggConfigs', visState.aggs)} | `; - } - - const schemas = getSchemas(vis, params.timeRange); - if (buildPipelineVisFunction[vis.type.name]) { - pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas, uiState); - } else if (vislibCharts.includes(vis.type.name)) { - const visConfig = visState.params; - visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); - - pipeline += `vislib ${prepareJson('visConfig', visState.params)}`; - } else { - pipeline += `visualization type='${vis.type.name}' - ${prepareJson('visConfig', visState.params)} - metricsAtAllLevels=${vis.isHierarchical()} - partialRows=${vis.params.showPartialRows || vis.type.name === 'tile_map'} `; - if (indexPattern) { - pipeline += `${prepareString('index', indexPattern.id)}`; - } - } - - return pipeline; -}; +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore +import { setBounds } from 'ui/agg_types/buckets/date_histogram'; +import { SearchSource } from 'ui/courier'; +import { AggConfig, Vis, VisState } from 'ui/vis'; + +interface SchemaFormat { + id: string; + params?: any; +} + +interface SchemaConfig { + accessor: number; + format: SchemaFormat | {}; + params: any; + aggType: string; +} + +interface Schemas { + metric: SchemaConfig[]; + bucket?: any[]; + geo_centroid?: any[]; + group?: any[]; + params?: any[]; + radius?: any[]; + segment?: any[]; + split_column?: any[]; + split_row?: any[]; + width?: any[]; + // catch all for schema name + [key: string]: any[] | undefined; +} + +type buildVisFunction = (visState: VisState, schemas: Schemas, uiState: any) => string; +type buildVisConfigFunction = (visState: Vis, schemas: Schemas) => VisState; + +interface BuildPipelineVisFunction { + [key: string]: buildVisFunction; +} + +interface BuildVisConfigFunction { + [key: string]: buildVisConfigFunction; +} + +const vislibCharts: string[] = [ + 'area', + 'gauge', + 'goal', + 'heatmap', + 'histogram', + 'horizontal_bar', + 'line', +]; + +export const getSchemas = (vis: Vis, timeRange?: any): Schemas => { + const createFormat = (agg: AggConfig): SchemaFormat => { + const format: SchemaFormat = agg.params.field ? agg.params.field.format.toJSON() : {}; + const formats: any = { + date_range: () => ({ id: 'string' }), + percentile_ranks: () => ({ id: 'percent' }), + count: () => ({ id: 'number' }), + cardinality: () => ({ id: 'number' }), + date_histogram: () => ({ + id: 'date', + params: { + pattern: agg.buckets.getScaledDateFormat(), + }, + }), + terms: () => ({ + id: 'terms', + params: { + id: format.id, + otherBucketLabel: agg.params.otherBucketLabel, + missingBucketLabel: agg.params.missingBucketLabel, + ...format.params, + }, + }), + range: () => ({ + id: 'range', + params: { id: format.id, ...format.params }, + }), + }; + + return formats[agg.type.name] ? formats[agg.type.name]() : format; + }; + + const createSchemaConfig = (accessor: number, agg: AggConfig): SchemaConfig => { + const schema = { + accessor, + format: {}, + params: {}, + aggType: agg.type.name, + }; + + if (agg.type.name === 'date_histogram') { + agg.params.timeRange = timeRange; + setBounds(agg, true); + } + if (agg.type.name === 'geohash_grid') { + schema.params = { + precision: agg.params.precision, + useGeocentroid: agg.params.useGeocentroid, + }; + } + + if ( + [ + 'derivative', + 'moving_avg', + 'serial_diff', + 'cumulative_sum', + 'sum_bucket', + 'avg_bucket', + 'min_bucket', + 'max_bucket', + ].includes(agg.type.name) + ) { + const subAgg = agg.params.customMetric || agg.aggConfigs.byId[agg.params.metricAgg]; + schema.format = createFormat(subAgg); + } else { + schema.format = createFormat(agg); + } + + return schema; + }; + + let cnt = 0; + const schemas: Schemas = { + metric: [], + }; + const responseAggs = vis.aggs.getResponseAggs().filter((agg: AggConfig) => agg.enabled); + const isHierarchical = vis.isHierarchical(); + const metrics = responseAggs.filter((agg: AggConfig) => agg.type.type === 'metrics'); + responseAggs.forEach((agg: AggConfig) => { + if (!agg.enabled) { + cnt++; + return; + } + let skipMetrics = false; + let schemaName = agg.schema ? agg.schema.name || agg.schema : null; + if (typeof schemaName === 'object') { + schemaName = null; + } + if (!schemaName) { + if (agg.type.name === 'geo_centroid') { + schemaName = 'geo_centroid'; + } else { + cnt++; + return; + } + } + if (schemaName === 'split') { + schemaName = `split_${agg.params.row ? 'row' : 'column'}`; + skipMetrics = true; + } + if (!schemas[schemaName]) { + schemas[schemaName] = []; + } + if (!isHierarchical || agg.type.type !== 'metrics') { + schemas[schemaName]!.push(createSchemaConfig(cnt++, agg)); + } + if (isHierarchical && (agg.type.type !== 'metrics' || metrics.length === responseAggs.length)) { + metrics.forEach((metric: any) => { + const schemaConfig = createSchemaConfig(cnt++, metric); + if (!skipMetrics) { + schemas.metric.push(schemaConfig); + } + }); + } + }); + return schemas; +}; + +export const prepareJson = (variable: string, data: object): string => { + return `${variable}='${JSON.stringify(data) + .replace(/\\/g, `\\\\`) + .replace(/'/g, `\\'`)}' `; +}; + +export const prepareString = (variable: string, data: string): string => { + return `${variable}='${data.replace(/\\/g, `\\\\`).replace(/'/g, `\\'`)}' `; +}; + +export const buildPipelineVisFunction: BuildPipelineVisFunction = { + vega: visState => { + return `vega ${prepareString('spec', visState.params.spec)}`; + }, + input_control_vis: visState => { + return `input_control_vis ${prepareJson('visConfig', visState.params)}`; + }, + metrics: (visState, schemas, uiState = {}) => { + const paramsJson = prepareJson('params', visState.params); + const uiStateJson = prepareJson('uiState', uiState); + + return `tsvb ${paramsJson} ${uiStateJson}`; + }, + timelion: visState => { + const expression = prepareString('expression', visState.params.expression); + const interval = prepareString('interval', visState.params.interval); + return `timelion_vis ${expression}${interval}`; + }, + markdown: visState => { + const expression = prepareString('expression', visState.params.markdown); + const visConfig = prepareJson('visConfig', visState.params); + return `kibana_markdown ${expression}${visConfig}`; + }, + table: (visState, schemas) => { + const visConfig = buildVisConfig.table(visState, schemas); + return `kibana_table ${prepareJson('visConfig', visConfig)}`; + }, + metric: (visState, schemas) => { + const visConfig = buildVisConfig.metric(visState, schemas); + return `kibana_metric ${prepareJson('visConfig', visConfig)}`; + }, + tagcloud: (visState, schemas) => { + const visConfig = buildVisConfig.tagcloud(visState, schemas); + return `tagcloud ${prepareJson('visConfig', visConfig)}`; + }, + region_map: (visState, schemas) => { + const visConfig = buildVisConfig.region_map(visState, schemas); + return `regionmap ${prepareJson('visConfig', visConfig)}`; + }, + tile_map: (visState, schemas) => { + const visConfig = buildVisConfig.tile_map(visState, schemas); + return `tilemap ${prepareJson('visConfig', visConfig)}`; + }, + pie: (visState, schemas) => { + const visConfig = buildVisConfig.pie(visState, schemas); + return `kibana_pie ${prepareJson('visConfig', visConfig)}`; + }, +}; + +const buildVisConfig: BuildVisConfigFunction = { + table: (visState, schemas) => { + const visConfig = visState.params; + visConfig.dimensions = { + metrics: schemas.metric, + buckets: schemas.bucket || [], + splitRow: schemas.split_row, + splitColumn: schemas.split_column, + }; + return visConfig; + }, + metric: (visState, schemas) => { + const visConfig = visState.params; + visConfig.metric.metrics = schemas.metric; + if (schemas.group) { + visConfig.metric.bucket = schemas.group[0]; + } + return visConfig; + }, + tagcloud: (visState, schemas) => { + const visConfig = visState.params; + visConfig.metric = schemas.metric[0]; + if (schemas.segment) { + visConfig.bucket = schemas.segment[0]; + } + return visConfig; + }, + region_map: (visState, schemas) => { + const visConfig = visState.params; + visConfig.metric = schemas.metric[0]; + if (schemas.segment) { + visConfig.bucket = schemas.segment[0]; + } + return visConfig; + }, + tile_map: (visState, schemas) => { + const visConfig = visState.params; + visConfig.dimensions = { + metric: schemas.metric[0], + geohash: schemas.segment ? schemas.segment[0] : null, + geocentroid: schemas.geo_centroid ? schemas.geo_centroid[0] : null, + }; + return visConfig; + }, + pie: (visState, schemas) => { + const visConfig = visState.params; + visConfig.dimensions = { + metric: schemas.metric[0], + buckets: schemas.segment, + splitRow: schemas.split_row, + splitColumn: schemas.split_column, + }; + return visConfig; + }, +}; + +export const buildVislibDimensions = (vis: any, timeRange?: any) => { + const schemas = getSchemas(vis, timeRange); + const dimensions = { + x: schemas.segment ? schemas.segment[0] : null, + y: schemas.metric, + z: schemas.radius, + width: schemas.width, + series: schemas.group, + splitRow: schemas.split_row, + splitColumn: schemas.split_column, + }; + if (schemas.segment) { + const xAgg = vis.aggs.getResponseAggs()[dimensions.x.accessor]; + if (xAgg.type.name === 'date_histogram') { + dimensions.x.params.date = true; + dimensions.x.params.interval = xAgg.buckets.getInterval().asMilliseconds(); + dimensions.x.params.format = xAgg.buckets.getScaledDateFormat(); + dimensions.x.params.bounds = xAgg.buckets.getBounds(); + } + } + + return dimensions; +}; + +// If not using the expression pipeline (i.e. visualize_data_loader), we need a mechanism to +// take a Vis object and decorate it with the necessary params (dimensions, bucket, metric, etc) +export const decorateVisObject = (vis: Vis, params: { timeRange?: any }) => { + const schemas = getSchemas(vis, params.timeRange); + let visConfig = vis.params; + if (buildVisConfig[vis.type.name]) { + visConfig = buildVisConfig[vis.type.name](vis, schemas); + vis.params = visConfig; + } else if (vislibCharts.includes(vis.type.name)) { + visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); + } +}; + +export const buildPipeline = ( + vis: Vis, + params: { searchSource: SearchSource; timeRange?: any } +) => { + const { searchSource } = params; + const { indexPattern } = vis; + const query = searchSource.getField('query'); + const filters = searchSource.getField('filter'); + const visState = vis.getCurrentState(); + const uiState = vis.getUiState(); + + // context + let pipeline = `kibana | kibana_context `; + if (query) { + pipeline += prepareJson('query', query); + } + if (filters) { + pipeline += prepareJson('filters', filters); + } + if (vis.savedSearchId) { + pipeline += prepareString('savedSearchId', vis.savedSearchId); + } + pipeline += '| '; + + // request handler + if (vis.type.requestHandler === 'courier') { + pipeline += `esaggs + ${prepareString('index', indexPattern.id)} + metricsAtAllLevels=${vis.isHierarchical()} + partialRows=${vis.params.showPartialRows || vis.type.requiresPartialRows || false} + ${prepareJson('aggConfigs', visState.aggs)} | `; + } + + const schemas = getSchemas(vis, params.timeRange); + if (buildPipelineVisFunction[vis.type.name]) { + pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas, uiState); + } else if (vislibCharts.includes(vis.type.name)) { + const visConfig = visState.params; + visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); + + pipeline += `vislib ${prepareJson('visConfig', visState.params)}`; + } else { + pipeline += `visualization type='${vis.type.name}' + ${prepareJson('visConfig', visState.params)} + metricsAtAllLevels=${vis.isHierarchical()} + partialRows=${vis.params.showPartialRows || vis.type.name === 'tile_map'} `; + if (indexPattern) { + pipeline += `${prepareString('index', indexPattern.id)}`; + } + } + + return pipeline; +}; From 97b6f56a23cf22478c6576343ae69a6fed9cdec6 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Fri, 15 Feb 2019 18:31:27 +0300 Subject: [PATCH 53/58] remove todo --- .../server/lib/vis_data/annorations/build_request_body.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js index 40c9a28563a642..ae592e432ab96b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js @@ -19,7 +19,6 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/annotations'; -//todo: (req, panel, annotation, esQueryConfig, indexPattern, capabilities export default function buildAnnotationRequest(...args) { const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); From de4b27c4222bded1750d39599b73acfa4d295e7d Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Sun, 17 Feb 2019 17:46:17 +0300 Subject: [PATCH 54/58] fix calculation of interval value for rollup search --- .../default_search_capabilities.js | 38 +++++--------- .../default_search_capabilities.test.js | 41 ---------------- .../default_search_strategy.test.js | 1 - .../lib/vis_data/helpers/unit_to_seconds.js | 25 +++++++++- .../rollup_search_capabilities.js | 27 +++------- .../rollup_search_capabilities.test.js | 49 +------------------ .../rollup_search_strategy.js | 6 +-- 7 files changed, 49 insertions(+), 138 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 334b3e210c7f04..86cf88a52808b5 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -16,14 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { INTERVAL_STRING_RE } from '../../../common/interval_regexp'; -import unitToSeconds from '../vis_data/helpers/unit_to_seconds'; - -const convertUnitToSeconds = interval => { - const matches = interval && interval.match(INTERVAL_STRING_RE); - - return matches ? Number(matches[1]) * unitToSeconds(matches[2]) : 0; -}; +import { convertIntervalToUnit, parseInterval } from '../vis_data/helpers/unit_to_seconds'; const getTimezoneFromRequest = request => { return request.payload.timerange.timezone; @@ -34,37 +27,32 @@ export default class DefaultSearchCapabilities { this.request = request; this.batchRequestsSupport = batchRequestsSupport; this.fieldsCapabilities = fieldsCapabilities; - this.validateTimeIntervalRules = []; - } - - get fixedTimeZone() { - return null; } get defaultTimeInterval() { return null; } - get defaultTimeIntervalInSeconds() { - return this.getIntervalInSeconds(this.defaultTimeInterval); - } - getSearchTimezone() { - return this.fixedTimeZone || getTimezoneFromRequest(this.request); + return getTimezoneFromRequest(this.request); } - getIntervalInSeconds(intervalString) { - return Boolean(intervalString) ? convertUnitToSeconds(intervalString) : 0; + parseInterval(interval) { + return parseInterval(interval); } - isTimeIntervalValid(intervalString) { - const userInterval = this.getIntervalInSeconds(intervalString); + convertIntervalToUnit(intervalString, unit) { + const parsedIntervalString = this.parseInterval(intervalString); + + if (parsedIntervalString.unit !== unit) { + return convertIntervalToUnit(intervalString, unit); + } - return this.validateTimeIntervalRules - .every(validationRule => validationRule(userInterval, this.defaultTimeIntervalInSeconds)); + return parsedIntervalString; } getValidTimeInterval(intervalString) { - return this.isTimeIntervalValid(intervalString) ? intervalString : this.defaultTimeInterval; + // Default search capabilities doesn't have any restrictions for the interval string + return intervalString; } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js index e28c7c4d9004d9..89b0713b70681c 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -33,25 +33,12 @@ describe('DefaultSearchCapabilities', () => { expect(defaultSearchCapabilities.request).toBe(req); expect(defaultSearchCapabilities.batchRequestsSupport).toBe(batchRequestsSupport); expect(defaultSearchCapabilities.fieldsCapabilities).toEqual({}); - expect(defaultSearchCapabilities.validateTimeIntervalRules).toEqual([]); - }); - - test('should return fixedTimeZone', () => { - expect(defaultSearchCapabilities.fixedTimeZone).toBe(null); }); test('should return defaultTimeInterval', () => { expect(defaultSearchCapabilities.defaultTimeInterval).toBe(null); }); - test('should return defaultTimeIntervalInSeconds', () => { - defaultSearchCapabilities.getIntervalInSeconds = jest.fn(() => '20m'); - - expect(defaultSearchCapabilities.defaultTimeIntervalInSeconds).toEqual('20m'); - expect(defaultSearchCapabilities.getIntervalInSeconds) - .toHaveBeenCalledWith(defaultSearchCapabilities.defaultTimeInterval); - }); - test('should return Search Timezone', () => { defaultSearchCapabilities.request = { payload: { @@ -64,35 +51,7 @@ describe('DefaultSearchCapabilities', () => { expect(defaultSearchCapabilities.getSearchTimezone()).toEqual('UTC'); }); - test('should return interval in seconds', () => { - expect(defaultSearchCapabilities.getIntervalInSeconds()).toEqual(0); - expect(defaultSearchCapabilities.getIntervalInSeconds('20m')).toEqual(1200); - expect(defaultSearchCapabilities.getIntervalInSeconds('1h')).toEqual(3600); - }); - - test('should check if a time interval is valid', () => { - defaultSearchCapabilities.validateTimeIntervalRules.push( - () => true - ); - - expect(defaultSearchCapabilities.isTimeIntervalValid()).toBe(true); - expect(defaultSearchCapabilities.isTimeIntervalValid('20m')).toBe(true); - expect(defaultSearchCapabilities.isTimeIntervalValid('1h')).toBe(true); - - defaultSearchCapabilities.validateTimeIntervalRules.push( - () => false - ); - - expect(defaultSearchCapabilities.isTimeIntervalValid('20m')).toBe(false); - }); - test('should return a valid time interval', () => { - defaultSearchCapabilities.isTimeIntervalValid = jest.fn(() => true); - expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe('20m'); - - defaultSearchCapabilities.isTimeIntervalValid = jest.fn(() => false); - - expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe(null); }); }); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js index 83930105b409dc..1d432e10a9cac5 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js @@ -64,7 +64,6 @@ describe('DefaultSearchStrategy', () => { request: req, batchRequestsSupport: true, fieldsCapabilities: {}, - validateTimeIntervalRules: [] }); }); }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js index dbbbd9a68f1d79..3aa27b40b44ae9 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { INTERVAL_STRING_RE } from '../../../../common/interval_regexp'; const units = { ms: 0.001, @@ -25,9 +26,31 @@ const units = { d: 86400, w: (86400) * 7, // Hum... might be wrong M: (86400) * 30, // this too... 29,30,31? - y: (86400) * 356 // Leap year? + y: (86400) * 356, // Leap year? +}; + +export const parseInterval = (intervalString) => { + let value; + let unit; + + if (intervalString) { + const matches = intervalString.match(INTERVAL_STRING_RE); + + value = Number(matches[1]); + unit = matches[2]; + } + + return { value, unit }; +}; + +export const convertIntervalToUnit = (intervalString, unit) => { + const parsedInterval = parseInterval(intervalString); + const value = Number((parsedInterval.value * units[parsedInterval.unit] / units[unit]).toFixed(2)); + + return { value, unit }; }; export default (unit) => { return units[unit]; }; + diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 4eafe215aff662..7d7f7bff58d397 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -5,8 +5,7 @@ */ import { get } from 'lodash'; -const intervalMultiple = (userTimeInterval, defaultTimeInterval) => !Boolean(userTimeInterval % defaultTimeInterval); -const roundN = (num, base) => Math.ceil(num / base) * base; +const leastCommonInterval = (num, base) => Math.max(Math.ceil(Math.floor(num) / base) * base, base); export default (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { @@ -14,23 +13,15 @@ export default (DefaultSearchCapabilities) => super(req, batchRequestsSupport, fieldsCapabilities); this.rollupIndex = rollupIndex; - this.init(); - } - - get fixedTimeZone() { - return get(this.dateHistogram, 'time_zone', null); + this.dateHistogram = this.getDateHistogramAggregation(); } get defaultTimeInterval() { return get(this.dateHistogram, 'interval', null); } - init() { - this.dateHistogram = this.getDateHistogramAggregation(); - - this.validateTimeIntervalRules = [ - intervalMultiple, - ]; + getSearchTimezone() { + return get(this.dateHistogram, 'time_zone', null); } getDateHistogramAggregation() { @@ -41,12 +32,10 @@ export default (DefaultSearchCapabilities) => } getValidTimeInterval(intervalString) { - if (this.isTimeIntervalValid(intervalString)) { - return intervalString; - } - - const userInterval = this.getIntervalInSeconds(intervalString); + const parsedDefaultInterval = this.parseInterval(this.defaultTimeInterval); + const parsedIntervalString = this.convertIntervalToUnit(intervalString, parsedDefaultInterval.unit); + const commonInterval = leastCommonInterval(parsedIntervalString.value, parsedDefaultInterval.value); - return `${roundN(userInterval, this.defaultTimeIntervalInSeconds)}s`; + return `${commonInterval}${parsedDefaultInterval.unit}`; } }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js index 30a60a9802cf5a..0b578a4800aba6 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -8,20 +8,6 @@ import getRollupSearchCapabilities from './rollup_search_capabilities'; class DefaultSearchCapabilities { constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { this.fieldsCapabilities = fieldsCapabilities; - this.validateTimeIntervalRules = []; - } - - get defaultTimeIntervalInSeconds() { - return this.getIntervalInSeconds(this.defaultTimeInterval); - } - - getIntervalInSeconds(intervalString) { - return Number.parseInt(intervalString); - } - - isTimeIntervalValid(interval) { - return this.validateTimeIntervalRules - .every(validationRule => validationRule(this.getIntervalInSeconds(interval), this.defaultTimeIntervalInSeconds)); } } @@ -56,48 +42,15 @@ describe('Rollup Search Capabilities', () => { test('should create instance of RollupSearchRequest', () => { expect(rollupSearchCaps).toBeInstanceOf(DefaultSearchCapabilities); - expect(rollupSearchCaps.isTimeIntervalValid).toBeDefined(); expect(rollupSearchCaps.fieldsCapabilities).toBe(fieldsCapabilities); expect(rollupSearchCaps.rollupIndex).toBe(rollupIndex); }); test('should return the "timezone" for the rollup request', () => { - expect(rollupSearchCaps.fixedTimeZone).toBe(testTimeZone); + expect(rollupSearchCaps.getSearchTimezone()).toBe(testTimeZone); }); test('should return the default "interval" for the rollup request', () => { expect(rollupSearchCaps.defaultTimeInterval).toBe(testInterval); }); - - describe('intervalMultiple', () => { - let intervalMultiple; - - beforeEach(() => { - [intervalMultiple] = rollupSearchCaps.validateTimeIntervalRules; - }); - - test('should add intervalMultiple into validation rules', () => { - expect(intervalMultiple).toBeDefined(); - }); - - test('should return true for multiple intervals', () => { - expect(intervalMultiple(6, 3)).toBeTruthy(); - expect(intervalMultiple(10, 5)).toBeTruthy(); - }); - - test('should return false for not multiple intervals', () => { - expect(intervalMultiple(7, 3)).toBeFalsy(); - expect(intervalMultiple(21, 10)).toBeFalsy(); - }); - }); - - describe('getValidTimeInterval', () => { - test('interval should be greater than default interval', () => { - expect(rollupSearchCaps.getValidTimeInterval('1s')).toBe(testInterval); - }); - - test('should round interval', () => { - expect(rollupSearchCaps.getValidTimeInterval('11s')).toBe('20s'); - }); - }); }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 6f1c627df393c7..825d4a9e2193e3 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -9,11 +9,11 @@ import mergeCapabilitiesWithFields from '../merge_capabilities_with_fields'; import { getCapabilitiesForRollupIndices } from '../map_capabilities'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; -const DEFAULT_INDEX_PATTERN = '*'; const batchRequestsSupport = false; const getRollupIndices = rollupData => Object.keys(rollupData); -const isIndexPatternValid = indexPattern => isString(indexPattern) && indexPattern !== DEFAULT_INDEX_PATTERN; +const isIndexPatternValid = indexPattern => indexPattern && + isString(indexPattern) && !indexPattern.includes('*'); export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { @@ -31,7 +31,7 @@ export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabil }).catch(() => Promise.resolve({})); } - async checkForViability(req, indexPattern = DEFAULT_INDEX_PATTERN) { + async checkForViability(req, indexPattern) { let isViable = false; let capabilities = null; From e3434dacb95351281ab1defa90da40639698b544 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 18 Feb 2019 11:49:15 +0300 Subject: [PATCH 55/58] add unit tests --- .../default_search_capabilities.test.js | 51 ++++++++++++++++++- .../lib/vis_data/helpers/unit_to_seconds.js | 6 +-- .../series/normalize_query.js | 8 +++ .../table/normalize_query.js | 3 ++ .../rollup_search_capabilities.test.js | 51 +++++++++++++++++++ .../rollup_search_strategy.js | 4 +- 6 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js index 89b0713b70681c..fa045b09f7ab34 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -20,8 +20,8 @@ import DefaultSearchCapabilities from './default_search_capabilities'; describe('DefaultSearchCapabilities', () => { let defaultSearchCapabilities; - let req; let batchRequestsSupport; + let req; beforeEach(() => { req = {}; @@ -54,4 +54,53 @@ describe('DefaultSearchCapabilities', () => { test('should return a valid time interval', () => { expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe('20m'); }); + + test('should parse interval', () => { + expect(defaultSearchCapabilities.parseInterval('120s')).toEqual({ + value: 120, + unit: 's' + }); + + expect(defaultSearchCapabilities.parseInterval('20m')).toEqual({ + value: 20, + unit: 'm' + }); + + expect(defaultSearchCapabilities.parseInterval('1y')).toEqual({ + value: 1, + unit: 'y' + }); + }); + + test('should convert interval string into different unit', () => { + expect(defaultSearchCapabilities.convertIntervalToUnit('120s', 's')).toEqual({ + value: 120, + unit: 's' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('60m', 'h')).toEqual({ + value: 1, + unit: 'h' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('4w', 'M')).toEqual({ + value: 1, + unit: 'M' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('1y', 'w')).toEqual({ + value: 48, + unit: 'w' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('60s', 'm')).toEqual({ + value: 1, + unit: 'm' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('1s', 'ms')).toEqual({ + value: 1000, + unit: 'ms' + }); + }); }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js index 3aa27b40b44ae9..4494883c44edc8 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js @@ -24,9 +24,9 @@ const units = { m: 60, h: 3600, d: 86400, - w: (86400) * 7, // Hum... might be wrong - M: (86400) * 30, // this too... 29,30,31? - y: (86400) * 356, // Leap year? + w: 86400 * 7, // Hum... might be wrong + M: 86400 * 7 * 4, // this too... 29,30,31? + y: 86400 * 7 * 4 * 12, // Leap year? }; export const parseInterval = (intervalString) => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js index 2683cd9b5c74e3..df2a829b90024d 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js @@ -20,6 +20,11 @@ const { set, get, isEmpty } = require('lodash'); const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); +/* For grouping by the 'Everything', the splitByEverything request processor + * creates fake .filter.match_all filter (see split_by_everything.js) to simplify the request processors code. + * But “filters” are not supported by all of available search strategies (e.g. Rollup search). + * This method removes that aggregation. + */ function removeEmptyTopLevelAggregation(doc, series) { const filter = get(doc, `aggs.${series.id}.filter`); @@ -32,6 +37,9 @@ function removeEmptyTopLevelAggregation(doc, series) { return doc; } +/* Last query handler in the chain. You can use this handler + * as the last place where you can modify the "doc" (request body) object before sending it to ES. + */ export default function normalizeQuery(req, panel, series) { return next => doc => { return next(removeEmptyTopLevelAggregation(doc, series)); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js index 2c561988b615ce..2d8eb281858bfe 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js @@ -20,6 +20,9 @@ const { set, get, isEmpty, forEach } = require('lodash'); const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); +/* Last query handler in the chain. You can use this handler + * as the last place where you can modify the "doc" (request body) object before sending it to ES. + */ export default function normalizeQuery() { return () => doc => { const series = get(doc, 'aggs.pivot.aggs'); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js index 0b578a4800aba6..a677c03ceee297 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -8,6 +8,7 @@ import getRollupSearchCapabilities from './rollup_search_capabilities'; class DefaultSearchCapabilities { constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { this.fieldsCapabilities = fieldsCapabilities; + this.parseInterval = jest.fn((interval) => interval); } } @@ -53,4 +54,54 @@ describe('Rollup Search Capabilities', () => { test('should return the default "interval" for the rollup request', () => { expect(rollupSearchCaps.defaultTimeInterval).toBe(testInterval); }); + + describe('getValidTimeInterval', () => { + let parsedDefaultInterval; + let parsedIntervalString; + + beforeEach(() => { + rollupSearchCaps.parseInterval = jest.fn(() => parsedDefaultInterval); + rollupSearchCaps.convertIntervalToUnit = jest.fn(() => parsedIntervalString); + }); + + test('should return 2y as common interval for 0.1y(user interval) and 2y(rollup interval)', () => { + parsedDefaultInterval = { + value: 2, + unit: 'y', + }; + parsedIntervalString = { + value: 0.1, + unit: 'y', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('2y'); + }); + + test('should return 3h as common interval for 2h(user interval) and 3h(rollup interval)', () => { + parsedDefaultInterval = { + value: 3, + unit: 'h', + }; + parsedIntervalString = { + value: 2, + unit: 'h', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('3h'); + }); + + test('should return 6m as common interval for 4m(user interval) and 3m(rollup interval)', () => { + parsedDefaultInterval = { + value: 3, + unit: 'm', + }; + parsedIntervalString = { + value: 4, + unit: 'm', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('6m'); + }); + }); + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index 825d4a9e2193e3..e9f9c34549a77a 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -12,8 +12,10 @@ const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; const batchRequestsSupport = false; const getRollupIndices = rollupData => Object.keys(rollupData); + +const isIndexPatternContainsWildcard = indexPattern => indexPattern.includes('*'); const isIndexPatternValid = indexPattern => indexPattern && - isString(indexPattern) && !indexPattern.includes('*'); + isString(indexPattern) && !isIndexPatternContainsWildcard(indexPattern); export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { From e4d92c9927a3d3412ccf17686ef94bb9a851b158 Mon Sep 17 00:00:00 2001 From: sulemanof Date: Mon, 18 Feb 2019 18:45:19 +0300 Subject: [PATCH 56/58] Remove default exports --- src/legacy/core_plugins/metrics/index.js | 2 +- .../metrics/server/lib/get_fields.js | 2 +- .../default_search_capabilities.js | 2 +- .../default_search_capabilities.test.js | 2 +- .../server/lib/search_strategies/index.js | 3 +- .../search_strategies_register.js | 10 +- .../search_strategies_register.test.js | 10 +- .../searh_requests/abstract_request.js | 2 +- .../searh_requests/abstract_request.test.js | 2 +- .../searh_requests/multi_search_request.js | 4 +- .../multi_search_request.test.js | 2 +- .../searh_requests/search_request.js | 8 +- .../searh_requests/search_request.test.js | 6 +- .../searh_requests/single_search_request.js | 4 +- .../single_search_request.test.js | 2 +- .../strategies/abstract_search_strategy.js | 2 +- .../abstract_search_strategy.test.js | 2 +- .../strategies/default_search_strategy.js | 8 +- .../default_search_strategy.test.js | 2 +- .../__tests__/helpers/get_es_shard_timeout.js | 2 +- .../annorations/build_request_body.js | 15 ++- .../annorations/get_request_params.js | 10 +- .../server/lib/vis_data/get_annotations.js | 108 +++++++++--------- .../server/lib/vis_data/get_series_data.js | 8 +- .../server/lib/vis_data/get_table_data.js | 2 +- .../vis_data/helpers/get_es_shard_timeout.js | 2 +- .../server/lib/vis_data/helpers/index.js | 2 - .../series/__tests__/date_histogram.js | 2 +- .../series/__tests__/build_request_body.js | 2 +- .../lib/vis_data/series/build_request_body.js | 17 ++- .../lib/vis_data/series/get_request_params.js | 8 +- x-pack/plugins/rollup/index.js | 2 +- .../lib/merge_capabilities_with_fields.js | 4 +- .../server/lib/search_strategies/index.js | 3 +- .../register_rollup_search_strategy.js | 8 +- .../register_rollup_search_strategy.test.js | 2 +- .../rollup_search_capabilities.js | 2 +- .../rollup_search_capabilities.test.js | 2 +- .../rollup_search_request.js | 2 +- .../rollup_search_request.test.js | 2 +- .../rollup_search_strategy.js | 4 +- .../rollup_search_strategy.test.js | 2 +- .../server/routes/api/index_patterns.js | 2 +- 43 files changed, 153 insertions(+), 135 deletions(-) diff --git a/src/legacy/core_plugins/metrics/index.js b/src/legacy/core_plugins/metrics/index.js index 358628e56d480f..b0b3dbb8377e0a 100644 --- a/src/legacy/core_plugins/metrics/index.js +++ b/src/legacy/core_plugins/metrics/index.js @@ -21,7 +21,7 @@ import { resolve } from 'path'; import fieldsRoutes from './server/routes/fields'; import visDataRoutes from './server/routes/vis'; -import SearchStrategiesRegister from './server/lib/search_strategies/search_strategies_register'; +import { SearchStrategiesRegister } from './server/lib/search_strategies/search_strategies_register'; export default function (kibana) { return new kibana.Plugin({ diff --git a/src/legacy/core_plugins/metrics/server/lib/get_fields.js b/src/legacy/core_plugins/metrics/server/lib/get_fields.js index 41e7f30be96457..d9830bca9b8507 100644 --- a/src/legacy/core_plugins/metrics/server/lib/get_fields.js +++ b/src/legacy/core_plugins/metrics/server/lib/get_fields.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import SearchStrategiesRegister from './search_strategies/search_strategies_register'; +import { SearchStrategiesRegister } from './search_strategies/search_strategies_register'; import { uniq } from 'lodash'; export async function getFields(req) { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 86cf88a52808b5..3381936535c78e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -22,7 +22,7 @@ const getTimezoneFromRequest = request => { return request.payload.timerange.timezone; }; -export default class DefaultSearchCapabilities { +export class DefaultSearchCapabilities { constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { this.request = request; this.batchRequestsSupport = batchRequestsSupport; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js index fa045b09f7ab34..af03c49fa973e1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import DefaultSearchCapabilities from './default_search_capabilities'; +import { DefaultSearchCapabilities } from './default_search_capabilities'; describe('DefaultSearchCapabilities', () => { let defaultSearchCapabilities; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js index ec54845490f921..512894f30a619d 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js @@ -16,6 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -import SearchStrategiesRegister from './search_strategies_register'; -export default SearchStrategiesRegister; +export { SearchStrategiesRegister } from './search_strategies_register'; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js index f577bb446b77b3..159e25191fc947 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchStrategy from './strategies/abstract_search_strategy'; -import AbstractSearchRequest from './searh_requests/abstract_request'; -import DefaultSearchStrategy from './strategies/default_search_strategy'; -import DefaultSearchCapabilities from './default_search_capabilities'; +import { AbstractSearchStrategy } from './strategies/abstract_search_strategy'; +import { AbstractSearchRequest } from './searh_requests/abstract_request'; +import { DefaultSearchStrategy } from './strategies/default_search_strategy'; +import { DefaultSearchCapabilities } from './default_search_capabilities'; const strategies = []; @@ -30,7 +30,7 @@ const addStrategy = searchStrategy => { return strategies; }; -export default class SearchStrategiesRegister { +export class SearchStrategiesRegister { static init(server) { server.expose('AbstractSearchStrategy', AbstractSearchStrategy); server.expose('AbstractSearchRequest', AbstractSearchRequest); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js index 891319ca696393..abe25b4737b7e1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import SearchStrategiesRegister from './search_strategies_register'; -import AbstractSearchStrategy from './strategies/abstract_search_strategy'; -import DefaultSearchStrategy from './strategies/default_search_strategy'; -import AbstractSearchRequest from './searh_requests/abstract_request'; -import DefaultSearchCapabilities from './default_search_capabilities'; +import { SearchStrategiesRegister } from './search_strategies_register'; +import { AbstractSearchStrategy } from './strategies/abstract_search_strategy'; +import { DefaultSearchStrategy } from './strategies/default_search_strategy'; +import { AbstractSearchRequest } from './searh_requests/abstract_request'; +import { DefaultSearchCapabilities } from './default_search_capabilities'; class MockSearchStrategy extends AbstractSearchStrategy { checkForViability() { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js index 2c1c4f707dbe75..cfacaf18a5abf1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -export default class AbstractSearchRequest { +export class AbstractSearchRequest { constructor(req, callWithRequest, indexPattern) { this.req = req; this.callWithRequest = callWithRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js index 4f8b10a2ef1248..16dbd9b580f695 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchRequest from './abstract_request'; +import { AbstractSearchRequest } from './abstract_request'; describe('AbstractSearchRequest', () => { let searchRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js index 663935b9371f22..793448073397bc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchRequest from './abstract_request'; +import { AbstractSearchRequest } from './abstract_request'; const SEARCH_METHOD = 'msearch'; -export default class MultiSearchRequest extends AbstractSearchRequest { +export class MultiSearchRequest extends AbstractSearchRequest { async search(options) { const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); const { responses } = await this.callWithRequest(this.req, SEARCH_METHOD, { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js index ef9c5fc32adf15..48d24f76227966 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import MultiSearchRequest from './multi_search_request'; +import { MultiSearchRequest } from './multi_search_request'; describe('MultiSearchRequest', () => { let searchRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js index 5347cb144840f0..afab9d37f30187 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchRequest from './abstract_request'; +import { AbstractSearchRequest } from './abstract_request'; -import MultiSearchRequest from './multi_search_request'; -import SingleSearchRequest from './single_search_request'; +import { MultiSearchRequest } from './multi_search_request'; +import { SingleSearchRequest } from './single_search_request'; -export default class SearchRequest extends AbstractSearchRequest { +export class SearchRequest extends AbstractSearchRequest { getSearchRequestType(options) { const isMultiSearch = Array.isArray(options.body); const SearchRequest = isMultiSearch ? MultiSearchRequest : SingleSearchRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js index e0999e9c91b71c..608f8abddf95d7 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -import SearchRequest from './search_request'; -import MultiSearchRequest from './multi_search_request'; -import SingleSearchRequest from './single_search_request'; +import { SearchRequest } from './search_request'; +import { MultiSearchRequest } from './multi_search_request'; +import { SingleSearchRequest } from './single_search_request'; describe('SearchRequest', () => { let searchRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js index 189b226638a563..df27b890f0fd00 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchRequest from './abstract_request'; +import { AbstractSearchRequest } from './abstract_request'; const SEARCH_METHOD = 'search'; -export default class SingleSearchRequest extends AbstractSearchRequest { +export class SingleSearchRequest extends AbstractSearchRequest { async search(options) { const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); const resp = await this.callWithRequest(this.req, SEARCH_METHOD, { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js index c10409f1acd4bd..97cbaa188cee4c 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import SingleSearchRequest from './single_search_request'; +import { SingleSearchRequest } from './single_search_request'; describe('SingleSearchRequest', () => { let searchRequest; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js index d11a20e1c45b43..378ff23fbe917c 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -export default class AbstractSearchStrategy { +export class AbstractSearchStrategy { constructor(server, callWithRequestFactory, SearchRequest) { this.getCallWithRequestInstance = req => callWithRequestFactory(server, req); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js index 3b04ae69f4014c..b3c3dddb8f8a3f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchStrategy from './abstract_search_strategy'; +import { AbstractSearchStrategy } from './abstract_search_strategy'; class SearchRequest { constructor(req, callWithRequest, indexPattern) { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js index a5f1f8d8a5e663..15a796b5e511a0 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -import AbstractSearchStrategy from './abstract_search_strategy'; -import SearchRequest from '../searh_requests/search_request'; -import DefaultSearchCapabilities from '../default_search_capabilities'; +import { AbstractSearchStrategy } from './abstract_search_strategy'; +import { SearchRequest } from '../searh_requests/search_request'; +import { DefaultSearchCapabilities } from '../default_search_capabilities'; const callWithRequestFactory = (server, request) => { const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); @@ -27,7 +27,7 @@ const callWithRequestFactory = (server, request) => { }; const batchRequestsSupport = true; -export default class DefaultSearchStrategy extends AbstractSearchStrategy { +export class DefaultSearchStrategy extends AbstractSearchStrategy { name = 'default'; constructor(server) { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js index 1d432e10a9cac5..9f1750c1b3db2b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import DefaultSearchStrategy from './default_search_strategy'; +import { DefaultSearchStrategy } from './default_search_strategy'; describe('DefaultSearchStrategy', () => { let defaultSearchStrategy; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js index aa4a15b8202477..4000fa06aa5c5e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js @@ -19,7 +19,7 @@ import sinon from 'sinon'; import { expect } from 'chai'; -import getEsShardTimeout from '../../helpers/get_es_shard_timeout'; +import { getEsShardTimeout } from '../../helpers/get_es_shard_timeout'; describe('getEsShardTimeout', () => { it('should return the elasticsearch.shardTimeout', () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js index ae592e432ab96b..91f55b1eb2200e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js @@ -19,7 +19,20 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/annotations'; -export default function buildAnnotationRequest(...args) { +/** + * Builds annotation request body + * + * @param {...args}: [ + * req: {Object} - a request object, + * panel: {Object} - a panel object, + * annotation: {Object} - an annotation object, + * esQueryConfig: {Object} - es query config object, + * indexPatternObject: {Object} - an index pattern object, + * capabilities: {Object} - a search capabilities object + * ] + * @returns {Object} doc - processed body + */ +export function buildAnnotationRequest(...args) { const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js index 81bac08831472f..c6fad7fa33dc6b 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -import buildRequestBody from './build_request_body'; -import getEsShardTimeout from '../helpers/get_es_shard_timeout'; +import { buildAnnotationRequest } from './build_request_body'; +import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; import { getIndexPatternObject } from '../helpers/get_index_pattern'; -export default async (req, panel, annotation, esQueryConfig, capabilities) => { +export async function getAnnotationRequestParams(req, panel, annotation, esQueryConfig, capabilities) { const bodies = []; const esShardTimeout = getEsShardTimeout(req); const indexPattern = annotation.index_pattern; const indexPatternObject = await getIndexPatternObject(req, indexPattern); - const request = buildRequestBody(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities); + const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities); if (capabilities.batchRequestsSupport) { bodies.push({ @@ -41,4 +41,4 @@ export default async (req, panel, annotation, esQueryConfig, capabilities) => { bodies.push(request); return bodies; -}; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index 56de63a554a7f1..16d8d5c4eed866 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -1,54 +1,54 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import handleAnnotationResponse from './handle_annotation_response'; -import getRequestParams from './annorations/get_request_params'; - -function validAnnotation(annotation) { - return annotation.index_pattern && - annotation.time_field && - annotation.fields && - annotation.icon && - annotation.template; -} - -export default async (req, panel, esQueryConfig, searchStrategy, capabilities) => { - const indexPattern = panel.index_pattern; - const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); - const annotations = panel.annotations.filter(validAnnotation); - - const bodiesPromises = annotations.map(annotation => getRequestParams(req, panel, annotation, esQueryConfig, capabilities)); - const body = (await Promise.all(bodiesPromises)) - .reduce((acc, items) => acc.concat(items), []); - - if (!body.length) return { responses: [] }; - - try { - const responses = await searchRequest.search({ body }); - - return annotations - .reduce((acc, annotation, index) => { - acc[annotation.id] = handleAnnotationResponse(responses[index], annotation); - - return acc; - }, {}); - } catch (error) { - if (error.message === 'missing-indices') return { responses: [] }; - throw error; - } -}; +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import handleAnnotationResponse from './handle_annotation_response'; +import { getAnnotationRequestParams } from './annorations/get_request_params'; + +function validAnnotation(annotation) { + return annotation.index_pattern && + annotation.time_field && + annotation.fields && + annotation.icon && + annotation.template; +} + +export async function getAnnotations(req, panel, esQueryConfig, searchStrategy, capabilities) { + const indexPattern = panel.index_pattern; + const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); + const annotations = panel.annotations.filter(validAnnotation); + + const bodiesPromises = annotations.map(annotation => getAnnotationRequestParams(req, panel, annotation, esQueryConfig, capabilities)); + const body = (await Promise.all(bodiesPromises)) + .reduce((acc, items) => acc.concat(items), []); + + if (!body.length) return { responses: [] }; + + try { + const responses = await searchRequest.search({ body }); + + return annotations + .reduce((acc, annotation, index) => { + acc[annotation.id] = handleAnnotationResponse(responses[index], annotation); + + return acc; + }, {}); + } catch (error) { + if (error.message === 'missing-indices') return { responses: [] }; + throw error; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index ff697452bcf9b0..7b1e278b8bdcf1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import getRequestParams from './series/get_request_params'; +import { getSeriesRequestParams } from './series/get_request_params'; import handleResponseBody from './series/handle_response_body'; import handleErrorResponse from './handle_error_response'; -import getAnnotations from './get_annotations'; -import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; +import { getAnnotations } from './get_annotations'; +import { SearchStrategiesRegister } from '../search_strategies/search_strategies_register'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; export async function getSeriesData(req, panel) { @@ -29,7 +29,7 @@ export async function getSeriesData(req, panel) { const searchRequest = searchStrategy.getSearchRequest(req, indexPattern); const esQueryConfig = await getEsQueryConfig(req); - const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig, capabilities)); + const bodiesPromises = panel.series.map(series => getSeriesRequestParams(req, panel, series, esQueryConfig, capabilities)); const body = (await Promise.all(bodiesPromises)) .reduce((acc, items) => acc.concat(items), []); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index 15e1cc9bc599e8..9d83e665b23b22 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -21,7 +21,7 @@ import buildRequestBody from './table/build_request_body'; import handleErrorResponse from './handle_error_response'; import { get } from 'lodash'; import processBucket from './table/process_bucket'; -import SearchStrategiesRegister from '../search_strategies/search_strategies_register'; +import { SearchStrategiesRegister } from '../search_strategies/search_strategies_register'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; import { getIndexPatternObject } from './helpers/get_index_pattern'; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js index 36636d8c57fdc5..839383b7d3695e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js @@ -17,6 +17,6 @@ * under the License. */ -export default function getEsShardTimeout(req) { +export function getEsShardTimeout(req) { return req.server.config().get('elasticsearch.shardTimeout'); } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js index f5159d8b2bf43b..01df1a6f54acdc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js @@ -22,7 +22,6 @@ import getAggValue from './get_agg_value'; import getBucketSize from './get_bucket_size'; import getBucketPath from './get_buckets_path'; import getDefaultDecoration from './get_default_decoration'; -import getEsShardTimeout from './get_es_shard_timeout'; import getLastMetric from './get_last_metric'; import getSiblingAggValue from './get_sibling_agg_value'; import getSplits from './get_splits'; @@ -37,7 +36,6 @@ export default { getBucketSize, getBucketPath, getDefaultDecoration, - getEsShardTimeout, getLastMetric, getSiblingAggValue, getSplits, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js index a1d16198112fa6..34bcf090fecf23 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js @@ -20,7 +20,7 @@ import dateHistogram from '../date_histogram'; import { expect } from 'chai'; import sinon from 'sinon'; -import DefaultSearchCapabilities from '../../../../search_strategies/default_search_capabilities'; +import { DefaultSearchCapabilities } from '../../../../search_strategies/default_search_capabilities'; describe('dateHistogram(req, panel, series)', () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index 17bdaf220c6622..cbeecc06596dab 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -77,7 +77,7 @@ const body = JSON.parse(` import sinon from 'sinon'; import { expect } from 'chai'; -import buildRequestBody from '../build_request_body'; +import { buildRequestBody } from '../build_request_body'; describe('buildRequestBody(req)', () => { it('returns a valid body', () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js index 57f49b923172a8..a5b724e11ef55a 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js @@ -20,10 +20,21 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/series'; -function buildRequestBody(...args) { +/** + * Builds series request body + * + * @param {...args}: [ + * req: {Object} - a request object, + * panel: {Object} - a panel object, + * series: {Object} - an series object, + * esQueryConfig: {Object} - es query config object, + * indexPatternObject: {Object} - an index pattern object, + * capabilities: {Object} - a search capabilities object + * ] + * @returns {Object} doc - processed body + */ +export function buildRequestBody(...args) { const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } - -export default buildRequestBody; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index d1f8cbfcae43ac..b957a34e15bba0 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import buildRequestBody from './build_request_body'; -import getEsShardTimeout from '../helpers/get_es_shard_timeout'; +import { buildRequestBody } from './build_request_body'; +import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; import { getIndexPatternObject } from '../helpers/get_index_pattern'; -export default async (req, panel, series, esQueryConfig, capabilities) => { +export async function getSeriesRequestParams(req, panel, series, esQueryConfig, capabilities) { const bodies = []; const indexPattern = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; const indexPatternObject = await getIndexPatternObject(req, indexPattern); @@ -40,4 +40,4 @@ export default async (req, panel, series, esQueryConfig, capabilities) => { bodies.push(request); return bodies; -}; +} diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index 6cfca8871bd3cf..ca8683673d261a 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -8,7 +8,7 @@ import { resolve } from 'path'; import { PLUGIN } from './common'; import { registerLicenseChecker } from './server/lib/register_license_checker'; import { rollupDataEnricher } from './rollup_data_enricher'; -import registerRollupSearchStrategy from './server/lib/search_strategies'; +import { registerRollupSearchStrategy } from './server/lib/search_strategies'; import { registerIndicesRoute, registerFieldsForWildcardRoute, diff --git a/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js index fe4541e676975a..bbad5d9e4e48e9 100644 --- a/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js +++ b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js @@ -6,7 +6,7 @@ // Merge rollup capabilities information with field information -const mergeCapabilitiesWithFields = (rollupIndexCapabilities, fieldsFromFieldCapsApi, previousFields = []) => { +export const mergeCapabilitiesWithFields = (rollupIndexCapabilities, fieldsFromFieldCapsApi, previousFields = []) => { const rollupFields = [...previousFields]; const rollupFieldNames = []; @@ -65,5 +65,3 @@ const mergeCapabilitiesWithFields = (rollupIndexCapabilities, fieldsFromFieldCap return rollupFields; }; - -export default mergeCapabilitiesWithFields; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/index.js b/x-pack/plugins/rollup/server/lib/search_strategies/index.js index 824d9fce2395f3..2d2711dfc932f4 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/index.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/index.js @@ -3,6 +3,5 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import registerRollupSearchStrategy from './register_rollup_search_strategy'; -export default registerRollupSearchStrategy; +export { registerRollupSearchStrategy } from './register_rollup_search_strategy'; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index 8dc1e4d154cc13..a7efed9850fc24 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -3,11 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import getRollupSearchStrategy from './rollup_search_strategy'; -import getRollupSearchRequest from './rollup_search_request'; -import getRollupSearchCapabilities from './rollup_search_capabilities'; +import { getRollupSearchStrategy } from './rollup_search_strategy'; +import { getRollupSearchRequest } from './rollup_search_request'; +import { getRollupSearchCapabilities } from './rollup_search_capabilities'; -export default (kbnServer, server) => kbnServer.afterPluginsInit(() => { +export const registerRollupSearchStrategy = (kbnServer, server) => kbnServer.afterPluginsInit(() => { if (!server.plugins.metrics) { return; } diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js index 488cf102b7dd8a..acd2d48c897066 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import registerRollupSearchStrategy from './register_rollup_search_strategy'; +import { registerRollupSearchStrategy } from './register_rollup_search_strategy'; describe('Register Rollup Search Strategy', () => { let kbnServer; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index 7d7f7bff58d397..f19a84b4553783 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -7,7 +7,7 @@ import { get } from 'lodash'; const leastCommonInterval = (num, base) => Math.max(Math.ceil(Math.floor(num) / base) * base, base); -export default (DefaultSearchCapabilities) => +export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { constructor(req, batchRequestsSupport, fieldsCapabilities, rollupIndex) { super(req, batchRequestsSupport, fieldsCapabilities); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js index a677c03ceee297..1c52b03057cfcf 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import getRollupSearchCapabilities from './rollup_search_capabilities'; +import { getRollupSearchCapabilities } from './rollup_search_capabilities'; class DefaultSearchCapabilities { constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js index c14df927fae79e..6ec8d9a8821687 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js @@ -5,7 +5,7 @@ */ const SEARCH_METHOD = 'rollup.search'; -export default (AbstractSearchRequest) => +export const getRollupSearchRequest = (AbstractSearchRequest) => (class RollupSearchRequest extends AbstractSearchRequest { async search(options) { const bodies = Array.isArray(options.body) ? options.body : [options.body]; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js index 71910e591f04b3..3c90cad4650e3f 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import getRollupSearchRequest from './rollup_search_request'; +import { getRollupSearchRequest } from './rollup_search_request'; class AbstractSearchRequest { indexPattern = 'indexPattern'; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js index e9f9c34549a77a..a38dc49cac9158 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -5,7 +5,7 @@ */ import { indexBy, isString } from 'lodash'; import { callWithRequestFactory } from '../call_with_request_factory'; -import mergeCapabilitiesWithFields from '../merge_capabilities_with_fields'; +import { mergeCapabilitiesWithFields } from '../merge_capabilities_with_fields'; import { getCapabilitiesForRollupIndices } from '../map_capabilities'; const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; @@ -17,7 +17,7 @@ const isIndexPatternContainsWildcard = indexPattern => indexPattern.includes('*' const isIndexPatternValid = indexPattern => indexPattern && isString(indexPattern) && !isIndexPatternContainsWildcard(indexPattern); -export default (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => +export const getRollupSearchStrategy = (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => (class RollupSearchStrategy extends AbstractSearchStrategy { name = 'rollup'; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js index 0cd2354d96a4cd..f13c2070f713a3 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import getRollupSearchStrategy from './rollup_search_strategy'; +import { getRollupSearchStrategy } from './rollup_search_strategy'; describe('Rollup Search Strategy', () => { let RollupSearchStrategy; diff --git a/x-pack/plugins/rollup/server/routes/api/index_patterns.js b/x-pack/plugins/rollup/server/routes/api/index_patterns.js index 2c2ae186252481..99fba02cc7b7d6 100644 --- a/x-pack/plugins/rollup/server/routes/api/index_patterns.js +++ b/x-pack/plugins/rollup/server/routes/api/index_patterns.js @@ -10,7 +10,7 @@ import { wrapEsError, wrapUnknownError } from '../../lib/error_wrappers'; import { licensePreRoutingFactory } from'../../lib/license_pre_routing_factory'; import indexBy from 'lodash/collection/indexBy'; import { getCapabilitiesForRollupIndices } from '../../lib/map_capabilities'; -import mergeCapabilitiesWithFields from '../../lib/merge_capabilities_with_fields'; +import { mergeCapabilitiesWithFields } from '../../lib/merge_capabilities_with_fields'; import querystring from 'querystring'; /** From 789a67c94e9632dff798b9511d10d4e398f33c65 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 19 Feb 2019 13:09:15 +0300 Subject: [PATCH 57/58] fix PR comments --- .../default_search_capabilities.js | 8 +++--- .../default_search_capabilities.test.js | 2 +- .../strategies/abstract_search_strategy.js | 5 ++++ .../abstract_search_strategy.test.js | 1 + .../annotations/date_histogram.js | 2 +- .../series/date_histogram.js | 2 +- .../series/__tests__/build_request_body.js | 5 ++-- .../rollup_search_capabilities.js | 28 +++++++++---------- .../rollup_search_capabilities.test.js | 2 +- 9 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js index 3381936535c78e..33e268577bf047 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -33,7 +33,7 @@ export class DefaultSearchCapabilities { return null; } - getSearchTimezone() { + get searchTimezone() { return getTimezoneFromRequest(this.request); } @@ -42,13 +42,13 @@ export class DefaultSearchCapabilities { } convertIntervalToUnit(intervalString, unit) { - const parsedIntervalString = this.parseInterval(intervalString); + const parsedInterval = this.parseInterval(intervalString); - if (parsedIntervalString.unit !== unit) { + if (parsedInterval.unit !== unit) { return convertIntervalToUnit(intervalString, unit); } - return parsedIntervalString; + return parsedInterval; } getValidTimeInterval(intervalString) { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js index af03c49fa973e1..5d41e03722d9db 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -48,7 +48,7 @@ describe('DefaultSearchCapabilities', () => { } }; - expect(defaultSearchCapabilities.getSearchTimezone()).toEqual('UTC'); + expect(defaultSearchCapabilities.searchTimezone).toEqual('UTC'); }); test('should return a valid time interval', () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js index 378ff23fbe917c..2df4652e8179c4 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + export class AbstractSearchStrategy { constructor(server, callWithRequestFactory, SearchRequest) { this.getCallWithRequestInstance = req => callWithRequestFactory(server, req); @@ -34,4 +35,8 @@ export class AbstractSearchStrategy { pattern: indexPattern, }); } + + checkForViability() { + throw new TypeError('Must override method'); + } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js index b3c3dddb8f8a3f..c063c5047d0694 100644 --- a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -54,6 +54,7 @@ describe('AbstractSearchStrategy', () => { expect(abstractSearchStrategy.getCallWithRequestInstance).toBeDefined(); expect(abstractSearchStrategy.getSearchRequest).toBeDefined(); expect(abstractSearchStrategy.getFieldsForWildcard).toBeDefined(); + expect(abstractSearchStrategy.checkForViability).toBeDefined(); }); test('should return fields for wildcard', async () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js index e9a488db23f7e3..05657842a5e2a7 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js @@ -25,7 +25,7 @@ export default function dateHistogram(req, panel, annotation, esQueryConfig, ind const timeField = annotation.time_field; const { bucketSize, intervalString } = getBucketSize(req, 'auto', capabilities); const { from, to } = getTimerange(req); - const timezone = capabilities.getSearchTimezone(); + const timezone = capabilities.searchTimezone; _.set(doc, `aggs.${annotation.id}.date_histogram`, { field: timeField, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js index 6c4934ebc2c157..c04684aab7e4f7 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js @@ -26,7 +26,7 @@ export default function dateHistogram(req, panel, series, esQueryConfig, indexPa const { timeField, interval } = getIntervalAndTimefield(panel, series); const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); const { from, to } = offsetTime(req, series.offset_time); - const timezone = capabilities.getSearchTimezone(); + const timezone = capabilities.searchTimezone; set(doc, `aggs.${series.id}.aggs.timeseries.date_histogram`, { field: timeField, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index cbeecc06596dab..195ae5b01c0f9f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -83,10 +83,9 @@ describe('buildRequestBody(req)', () => { it('returns a valid body', () => { const panel = body.panels[0]; const series = panel.series[0]; - const getSearchTimezone = sinon.spy(() => 'UTC'); const getValidTimeInterval = sinon.spy(() => '10s'); const capabilities = { - getSearchTimezone, + searchTimezone: 'UTC', getValidTimeInterval }; const config = { @@ -95,7 +94,7 @@ describe('buildRequestBody(req)', () => { }; const indexPatternObject = {}; const doc = buildRequestBody({ payload: body }, panel, series, config, indexPatternObject, capabilities); - expect(getSearchTimezone.calledOnce).to.equal(true); + expect(doc).to.eql({ size: 0, query: { diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index f19a84b4553783..ee38a5973c060a 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -5,7 +5,14 @@ */ import { get } from 'lodash'; -const leastCommonInterval = (num, base) => Math.max(Math.ceil(Math.floor(num) / base) * base, base); +const leastCommonInterval = (num = 0, base = 0) => Math.max(Math.ceil(num / base) * base, base); + +const getDateHistogramAggregation = (fieldsCapabilities, rollupIndex) => { + const dateHistogramField = fieldsCapabilities[rollupIndex].aggs.date_histogram; + + // there is also only one valid date_histogram field + return Object.values(dateHistogramField)[0]; +}; export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { @@ -13,29 +20,22 @@ export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => super(req, batchRequestsSupport, fieldsCapabilities); this.rollupIndex = rollupIndex; - this.dateHistogram = this.getDateHistogramAggregation(); + this.dateHistogram = getDateHistogramAggregation(fieldsCapabilities, rollupIndex); } get defaultTimeInterval() { return get(this.dateHistogram, 'interval', null); } - getSearchTimezone() { + get searchTimezone() { return get(this.dateHistogram, 'time_zone', null); } - getDateHistogramAggregation() { - const dateHistogramField = this.fieldsCapabilities[this.rollupIndex].aggs.date_histogram; - - // there is also only one valid date_histogram field - return Object.values(dateHistogramField)[0]; - } - getValidTimeInterval(intervalString) { - const parsedDefaultInterval = this.parseInterval(this.defaultTimeInterval); - const parsedIntervalString = this.convertIntervalToUnit(intervalString, parsedDefaultInterval.unit); - const commonInterval = leastCommonInterval(parsedIntervalString.value, parsedDefaultInterval.value); + const { unit, value } = this.parseInterval(this.defaultTimeInterval); + const parsedIntervalString = this.convertIntervalToUnit(intervalString, unit); + const commonInterval = leastCommonInterval(parsedIntervalString.value, value); - return `${commonInterval}${parsedDefaultInterval.unit}`; + return `${commonInterval}${unit}`; } }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js index 1c52b03057cfcf..a4033de8d60695 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -48,7 +48,7 @@ describe('Rollup Search Capabilities', () => { }); test('should return the "timezone" for the rollup request', () => { - expect(rollupSearchCaps.getSearchTimezone()).toBe(testTimeZone); + expect(rollupSearchCaps.searchTimezone).toBe(testTimeZone); }); test('should return the default "interval" for the rollup request', () => { From 623b900d8df95e87d3377c5838d996c9e10ca9a7 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 19 Feb 2019 18:40:11 +0300 Subject: [PATCH 58/58] fix calendar intervals --- .../rollup_search_capabilities.js | 22 +++++-- .../rollup_search_capabilities.test.js | 58 ++++++++++++++++--- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js index ee38a5973c060a..3e26b5c9b8532a 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; +import { unitsMap } from '@elastic/datemath'; const leastCommonInterval = (num = 0, base = 0) => Math.max(Math.ceil(num / base) * base, base); @@ -14,6 +15,8 @@ const getDateHistogramAggregation = (fieldsCapabilities, rollupIndex) => { return Object.values(dateHistogramField)[0]; }; +const isCalendarInterval = ({ unit, value }) => value === 1 && ['calendar', 'mixed'].includes(unitsMap[unit].type); + export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => (class RollupSearchCapabilities extends DefaultSearchCapabilities { constructor(req, batchRequestsSupport, fieldsCapabilities, rollupIndex) { @@ -31,11 +34,20 @@ export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => return get(this.dateHistogram, 'time_zone', null); } - getValidTimeInterval(intervalString) { - const { unit, value } = this.parseInterval(this.defaultTimeInterval); - const parsedIntervalString = this.convertIntervalToUnit(intervalString, unit); - const commonInterval = leastCommonInterval(parsedIntervalString.value, value); + getValidTimeInterval(userIntervalString) { + const parsedRollupJobInterval = this.parseInterval(this.defaultTimeInterval); + const parsedUserInterval = this.parseInterval(userIntervalString); + + let { unit } = parsedRollupJobInterval; + let { value } = this.convertIntervalToUnit(userIntervalString, unit); + + if (isCalendarInterval(parsedRollupJobInterval) && isCalendarInterval(parsedUserInterval)) { + unit = value > 1 ? parsedUserInterval.unit : parsedRollupJobInterval.unit; + value = 1; + } else { + value = leastCommonInterval(value, parsedRollupJobInterval.value); + } - return `${commonInterval}${unit}`; + return `${value}${unit}`; } }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js index a4033de8d60695..39906783fa1fa1 100644 --- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -57,19 +57,59 @@ describe('Rollup Search Capabilities', () => { describe('getValidTimeInterval', () => { let parsedDefaultInterval; - let parsedIntervalString; + let parsedUserIntervalString; + let convertedIntervalIntoDefaultUnit; beforeEach(() => { - rollupSearchCaps.parseInterval = jest.fn(() => parsedDefaultInterval); - rollupSearchCaps.convertIntervalToUnit = jest.fn(() => parsedIntervalString); + convertedIntervalIntoDefaultUnit = null; + + rollupSearchCaps.parseInterval = jest.fn() + .mockImplementationOnce(() => parsedDefaultInterval) + .mockImplementationOnce(() => parsedUserIntervalString); + rollupSearchCaps.convertIntervalToUnit = jest + .fn(() => convertedIntervalIntoDefaultUnit || parsedUserIntervalString); + }); + + test('should return 1w as common interval for 1w(user interval) and 1d(rollup interval) - calendar intervals', () => { + parsedDefaultInterval = { + value: 1, + unit: 'd', + }; + parsedUserIntervalString = { + value: 1, + unit: 'w', + }; + convertedIntervalIntoDefaultUnit = { + value: 7, + unit: 'd', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('1w'); + }); + + test('should return 1w as common interval for 1d(user interval) and 1w(rollup interval) - calendar intervals', () => { + parsedDefaultInterval = { + value: 1, + unit: 'w', + }; + parsedUserIntervalString = { + value: 1, + unit: 'd', + }; + convertedIntervalIntoDefaultUnit = { + value: 1 / 7, + unit: 'w', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('1w'); }); - test('should return 2y as common interval for 0.1y(user interval) and 2y(rollup interval)', () => { + test('should return 2y as common interval for 0.1y(user interval) and 2y(rollup interval) - fixed intervals', () => { parsedDefaultInterval = { value: 2, unit: 'y', }; - parsedIntervalString = { + parsedUserIntervalString = { value: 0.1, unit: 'y', }; @@ -77,12 +117,12 @@ describe('Rollup Search Capabilities', () => { expect(rollupSearchCaps.getValidTimeInterval()).toBe('2y'); }); - test('should return 3h as common interval for 2h(user interval) and 3h(rollup interval)', () => { + test('should return 3h as common interval for 2h(user interval) and 3h(rollup interval) - fixed intervals', () => { parsedDefaultInterval = { value: 3, unit: 'h', }; - parsedIntervalString = { + parsedUserIntervalString = { value: 2, unit: 'h', }; @@ -90,12 +130,12 @@ describe('Rollup Search Capabilities', () => { expect(rollupSearchCaps.getValidTimeInterval()).toBe('3h'); }); - test('should return 6m as common interval for 4m(user interval) and 3m(rollup interval)', () => { + test('should return 6m as common interval for 4m(user interval) and 3m(rollup interval) - fixed intervals', () => { parsedDefaultInterval = { value: 3, unit: 'm', }; - parsedIntervalString = { + parsedUserIntervalString = { value: 4, unit: 'm', };