From 17fc80cb4e15df72bd3adc8f66dfef723373c470 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 23 Jan 2019 21:03:43 +0000 Subject: [PATCH 1/3] [ML] Adding ability to override number of sample lines in file data viz --- .../components/edit_flyout/edit_flyout.js | 12 +- .../components/edit_flyout/overrides.js | 113 ++++++++++++++---- .../file_datavisualizer_view.js | 3 +- .../components/utils/overrides.js | 2 + .../components/utils/utils.js | 11 +- .../ml/server/client/elasticsearch_ml.js | 5 +- 6 files changed, 115 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/edit_flyout.js b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/edit_flyout.js index 5541603052516d..39da74850fc7e5 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/edit_flyout.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/edit_flyout.js @@ -29,6 +29,9 @@ export class EditFlyout extends Component { super(props); this.applyOverrides = () => {}; + this.state = { + overridesValid: true + }; } applyAndClose = () => { @@ -42,10 +45,14 @@ export class EditFlyout extends Component { unsetApplyOverrides = () => { this.applyOverrides = () => {}; } + setOverridesValid = (overridesValid) => { + this.setState({ overridesValid }); + } render() { - const { isFlyoutVisible, closeEditFlyout } = this.props; const { + isFlyoutVisible, + closeEditFlyout, setOverrides, overrides, originalSettings, @@ -78,6 +85,7 @@ export class EditFlyout extends Component { overrides={overrides} originalSettings={originalSettings} setApplyOverrides={this.setApplyOverrides} + setOverridesValid={this.setOverridesValid} fields={fields} /> @@ -105,8 +113,8 @@ export class EditFlyout extends Component { { - const overrides = { ...this.state }; - overrides.delimiter = convertDelimiterBack(overrides); - delete overrides.customDelimiter; - delete overrides.originalColumnNames; + const overrides = { ...this.state.overrides }; + overrides.delimiter = convertDelimiterBack(overrides.delimiter, this.state.customDelimiter); this.props.setOverrides(overrides); } + setOverride(o) { + const overrides = { ...this.state.overrides, ...o }; + this.setState({ overrides }); + } + onFormatChange = (format) => { - this.setState({ format }); + this.setOverride({ format }); } onTimestampFormatChange = (timestampFormat) => { - this.setState({ timestampFormat }); + this.setOverride({ timestampFormat }); } onTimestampFieldChange = (timestampField) => { - this.setState({ timestampField }); + this.setOverride({ timestampField }); } onDelimiterChange = (delimiter) => { - this.setState({ delimiter }); + this.setOverride({ delimiter }); } onCustomDelimiterChange = (e) => { @@ -128,54 +150,89 @@ export class Overrides extends Component { } onQuoteChange = (quote) => { - this.setState({ quote }); + this.setOverride({ quote }); } onHasHeaderRowChange = (e) => { - this.setState({ hasHeaderRow: e.target.checked }); + this.setOverride({ hasHeaderRow: e.target.checked }); } onShouldTrimFieldsChange = (e) => { - this.setState({ shouldTrimFields: e.target.checked }); + this.setOverride({ shouldTrimFields: e.target.checked }); } onCharsetChange = (charset) => { - this.setState({ charset }); + this.setOverride({ charset }); } onColumnNameChange = (e, i) => { - const columnNames = this.state.columnNames; + const columnNames = this.state.overrides.columnNames; columnNames[i] = e.target.value; - this.setState({ columnNames }); + this.setOverride({ columnNames }); } grokPatternChange = (e) => { - this.setState({ grokPattern: e.target.value }); + this.setOverride({ grokPattern: e.target.value }); + } + + onLinesToSampleChange = (e) => { + const linesToSample = +e.target.value; + this.setOverride({ linesToSample }); + + // check whether the value is valid and set that to state. + const linesToSampleValid = isLinesToSampleValid(linesToSample); + this.setState({ linesToSampleValid }); + + // set the overrides valid setting in the parent component, + // used to disable the Apply button if any of the overrides are invalid + this.props.setOverridesValid(linesToSampleValid); } render() { const { fields } = this.props; + const { + customDelimiter, + originalColumnNames, + linesToSampleValid, + } = this.state; + const { timestampFormat, timestampField, format, delimiter, - customDelimiter, quote, hasHeaderRow, shouldTrimFields, // charset, columnNames, - originalColumnNames, grokPattern, - } = this.state; + linesToSample, + } = this.state.overrides; const fieldOptions = fields.map(f => ({ value: f, inputDisplay: f })); return ( + + } + > + + + { - (this.state.format === 'delimited') && + (format === 'delimited') && } { - (this.state.format === 'semi_structured_text') && + (format === 'semi_structured_text') && */} { - (this.state.format === 'delimited' && originalColumnNames.length > 0) && + (format === 'delimited' && originalColumnNames.length > 0) && @@ -398,7 +455,7 @@ function convertDelimiter(d) { } // Convert the delimiter textual descriptions back to their real characters. -function convertDelimiterBack({ delimiter, customDelimiter }) { +function convertDelimiterBack(delimiter, customDelimiter) { switch (delimiter) { case 'comma': return ','; @@ -429,3 +486,7 @@ function getColumnNames(columnNames, originalSettings) { originalColumnNames, }; } + +function isLinesToSampleValid(linesToSample) { + return (linesToSample > LINES_TO_SAMPLE_VALUE_MIN && linesToSample <= LINES_TO_SAMPLE_VALUE_MAX); +} diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/file_datavisualizer_view/file_datavisualizer_view.js b/x-pack/plugins/ml/public/file_datavisualizer/components/file_datavisualizer_view/file_datavisualizer_view.js index 74d0a0ad4fd849..6c8f8046342737 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/file_datavisualizer_view/file_datavisualizer_view.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/file_datavisualizer_view/file_datavisualizer_view.js @@ -150,6 +150,7 @@ export class FileDataVisualizerView extends Component { } if (serverOverrides === undefined) { + // if no overrides were used, store all the settings returned from the endpoint this.originalSettings = serverSettings; } else { Object.keys(serverOverrides).forEach((o) => { @@ -164,7 +165,7 @@ export class FileDataVisualizerView extends Component { Object.keys(serverSettings).forEach((o) => { const value = serverSettings[o]; if ( - this.overrides[o] === undefined && + (this.overrides[o] === undefined) && (Array.isArray(value) && (isEqual(value, this.originalSettings[o]) === false) || (value !== this.originalSettings[o])) ) { diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/utils/overrides.js b/x-pack/plugins/ml/public/file_datavisualizer/components/utils/overrides.js index 413c2ca7372dcf..f8a90c87b9dc8d 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/utils/overrides.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/utils/overrides.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export const DEFAULT_LINES_TO_SAMPLE = 1000; export const overrideDefaults = { timestampFormat: undefined, @@ -16,4 +17,5 @@ export const overrideDefaults = { columnNames: undefined, shouldTrimFields: undefined, grokPattern: undefined, + linesToSample: undefined, }; diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/utils/utils.js b/x-pack/plugins/ml/public/file_datavisualizer/components/utils/utils.js index af9b24ba6e810f..c373db4a4a83f7 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/utils/utils.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/utils/utils.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { overrideDefaults } from './overrides'; +import { overrideDefaults, DEFAULT_LINES_TO_SAMPLE } from './overrides'; import { isEqual } from 'lodash'; import { ml } from '../../../services/ml_api_service'; @@ -86,6 +86,11 @@ export function createUrlOverrides(overrides, originalSettings) { if (formattedOverrides.grok_pattern !== '') { formattedOverrides.grok_pattern = encodeURIComponent(formattedOverrides.grok_pattern); } + + if (formattedOverrides.lines_to_sample === '') { + formattedOverrides.lines_to_sample = overrides.linesToSample; + } + return formattedOverrides; } @@ -93,6 +98,9 @@ export function processResults(results) { const timestampFormat = (results.joda_timestamp_formats !== undefined && results.joda_timestamp_formats.length) ? results.joda_timestamp_formats[0] : undefined; + const linesToSample = (results.overrides !== undefined && results.overrides.lines_to_sample !== undefined) ? + results.overrides.lines_to_sample : DEFAULT_LINES_TO_SAMPLE; + return { format: results.format, delimiter: results.delimiter, @@ -104,6 +112,7 @@ export function processResults(results) { charset: results.charset, columnNames: results.column_names, grokPattern: results.grok_pattern, + linesToSample, }; } diff --git a/x-pack/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/plugins/ml/server/client/elasticsearch_ml.js index 2342bafd3c5fb0..47574fb6576c5b 100644 --- a/x-pack/plugins/ml/server/client/elasticsearch_ml.js +++ b/x-pack/plugins/ml/server/client/elasticsearch_ml.js @@ -570,7 +570,7 @@ export const elasticsearchJsPlugin = (Client, config, components) => { urls: [ { // eslint-disable-next-line max-len - fmt: '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>', + fmt: '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', req: { charset: { type: 'string' @@ -602,6 +602,9 @@ export const elasticsearchJsPlugin = (Client, config, components) => { timestamp_format: { type: 'string' }, + lines_to_sample: { + type: 'string' + }, } }, { From 42757f1d33328099bd915a50af1d55c63ea2b85c Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 23 Jan 2019 21:13:20 +0000 Subject: [PATCH 2/3] tiny tweak --- .../file_datavisualizer/components/edit_flyout/overrides.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js index 58fe95d4be2420..9db81fd7d367c4 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js @@ -195,6 +195,7 @@ export class Overrides extends Component { customDelimiter, originalColumnNames, linesToSampleValid, + overrides, } = this.state; const { @@ -209,7 +210,7 @@ export class Overrides extends Component { columnNames, grokPattern, linesToSample, - } = this.state.overrides; + } = overrides; const fieldOptions = fields.map(f => ({ value: f, inputDisplay: f })); From 4ae81431ba1107e436d22688cb515e4bb86ae3a5 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 24 Jan 2019 08:51:08 +0000 Subject: [PATCH 3/3] updating tests --- .../__snapshots__/overrides.test.js.snap | 22 +++++++++++++++++++ .../components/edit_flyout/overrides.test.js | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap index eb9a4b8def88d2..134593d713243e 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap @@ -2,6 +2,28 @@ exports[`Overrides render overrides 1`] = ` + + } + > + + { ); - expect(component.state('format')).toEqual(FORMAT_1); + expect(component.state('overrides').format).toEqual(FORMAT_1); component.instance().onFormatChange(FORMAT_2); - expect(component.state('format')).toEqual(FORMAT_2); + expect(component.state('overrides').format).toEqual(FORMAT_2); }); });