From 7dbef2fb9a25a3122ceab828a1096766d4cc9636 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Tue, 12 Jan 2016 13:04:54 -0500 Subject: [PATCH 01/19] Starting work on pattern review page --- .../data/directives/pattern_review_step.html | 7 ++++++ .../data/directives/pattern_review_step.js | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html index 75d8dc49dab09d..3d6e318795d723 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html @@ -1,5 +1,12 @@

Pattern review step

+
+ + +
+
Docs: {{docs}}
diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index 6ceb1ecb03a13d..031345e3f33d9b 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -1,5 +1,17 @@ var modules = require('ui/modules'); var template = require('plugins/kibana/settings/sections/data/directives/pattern_review_step.html'); +var _ = require('lodash'); + +var testData = { + message: 'src=1.1.1.1 evil=1', + src: '1.1.1.1', + evil: '1', + coordinates: { + lat: 37.3894, + lon: 122.0819 + }, + '@timestamp': '2015-11-24T00:00:00.000Z' +}; modules.get('apps/settings') .directive('patternReviewStep', function () { @@ -8,6 +20,18 @@ modules.get('apps/settings') scope: { docs: '=', save: '&onSave' + }, + controller: function ($scope, Private) { + $scope.docs = testData; + $scope.columns = [ + {title: 'Field'}, + {title: 'Type'}, + {title: 'Example', sortable: false} + ]; + + $scope.rows = _.map($scope.docs, (value, key) => { + return [key, 'type', value]; + }); } }; }); From d4aff7709531a133b7dad83eefc6e716eaa81e76 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Tue, 12 Jan 2016 14:29:12 -0500 Subject: [PATCH 02/19] Type selection dropdown --- .../data/directives/pattern_review_step.html | 3 ++- .../data/directives/pattern_review_step.js | 16 ++++++++++++++-- .../sections/data/partials/_edit_field_type.html | 9 +++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html index 3d6e318795d723..cf394b42de50a0 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html @@ -11,4 +11,5 @@

Pattern review step

Docs: {{docs}} - + + diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index 031345e3f33d9b..d06e30e20c55ba 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -1,6 +1,7 @@ var modules = require('ui/modules'); var template = require('plugins/kibana/settings/sections/data/directives/pattern_review_step.html'); var _ = require('lodash'); +var editFieldTypeHTML = require('plugins/kibana/settings/sections/data/partials/_edit_field_type.html'); var testData = { message: 'src=1.1.1.1 evil=1', @@ -23,14 +24,25 @@ modules.get('apps/settings') }, controller: function ($scope, Private) { $scope.docs = testData; + $scope.fields = _.map($scope.docs, (value, key) => { + return {name: key, type: typeof value}; + }); + $scope.columns = [ {title: 'Field'}, {title: 'Type'}, {title: 'Example', sortable: false} ]; - $scope.rows = _.map($scope.docs, (value, key) => { - return [key, 'type', value]; + $scope.rows = _.map($scope.fields, (field) => { + return [ + field.name, + { + markup: editFieldTypeHTML, + scope: _.assign($scope.$new(), {field: field}) + }, + $scope.docs[field.name] + ]; }); } }; diff --git a/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html b/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html new file mode 100644 index 00000000000000..eecf3a3f35481f --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html @@ -0,0 +1,9 @@ + From 27e402bad468fd896ffbb0018bc35d059a8be1fc Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Tue, 12 Jan 2016 14:59:55 -0500 Subject: [PATCH 03/19] add index pattern input --- .../sections/data/directives/pattern_review_step.html | 11 +++++------ .../sections/data/directives/pattern_review_step.js | 9 +++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html index cf394b42de50a0..e0d52236a4359a 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html @@ -1,5 +1,9 @@

Pattern review step

+
+ +
+
Pattern review step
-
- Docs: {{docs}} -
- - - + diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index d06e30e20c55ba..c1695f532cbd56 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -24,17 +24,22 @@ modules.get('apps/settings') }, controller: function ($scope, Private) { $scope.docs = testData; - $scope.fields = _.map($scope.docs, (value, key) => { + $scope.indexPattern = {id: 'filebeat-*', title: 'filebeat-*'}; + $scope.indexPattern.fields = _.map($scope.docs, (value, key) => { return {name: key, type: typeof value}; }); + $scope.$watch('indexPattern.id', function (value) { + $scope.indexPattern.title = value; + }); + $scope.columns = [ {title: 'Field'}, {title: 'Type'}, {title: 'Example', sortable: false} ]; - $scope.rows = _.map($scope.fields, (field) => { + $scope.rows = _.map($scope.indexPattern.fields, (field) => { return [ field.name, { From 027e807eddb6a1812184419eec1ccead6080a182 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Tue, 12 Jan 2016 18:52:59 -0500 Subject: [PATCH 04/19] avoid overwriting pattern properties if they were previously set --- .../sections/data/directives/pattern_review_step.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index e44ee5a31d196e..dc96296352daed 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -25,10 +25,14 @@ modules.get('apps/settings') }, controller: function ($scope, Private) { $scope.sampleDocs = testData; - $scope.indexPattern = {id: 'filebeat-*', title: 'filebeat-*'}; - $scope.indexPattern.fields = _.map($scope.sampleDocs, (value, key) => { - return {name: key, type: typeof value}; - }); + if (_.isEmpty($scope.indexPattern)) { + $scope.indexPattern = {id: 'filebeat-*', title: 'filebeat-*'}; + } + if (_.isEmpty($scope.indexPattern.fields)) { + $scope.indexPattern.fields = _.map($scope.sampleDocs, (value, key) => { + return {name: key, type: typeof value}; + }); + } $scope.$watch('indexPattern.id', function (value) { $scope.indexPattern.title = value; From da6183cd6ede970923ddeca6f96cec8f3206f8d8 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Tue, 12 Jan 2016 19:07:09 -0500 Subject: [PATCH 05/19] add timeFieldName form input and time based checkbox --- .../data/directives/pattern_review_step.html | 10 +++++++++- .../data/directives/pattern_review_step.js | 18 ++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html index 61951167d55ce2..aa4f427dbfb379 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html @@ -1,7 +1,15 @@

Pattern review step

- + + +
diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index dc96296352daed..935314a3788b13 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -25,14 +25,20 @@ modules.get('apps/settings') }, controller: function ($scope, Private) { $scope.sampleDocs = testData; - if (_.isEmpty($scope.indexPattern)) { - $scope.indexPattern = {id: 'filebeat-*', title: 'filebeat-*'}; + if (_.isUndefined($scope.indexPattern)) { + $scope.indexPattern = {}; } - if (_.isEmpty($scope.indexPattern.fields)) { - $scope.indexPattern.fields = _.map($scope.sampleDocs, (value, key) => { + + _.defaults($scope.indexPattern, { + id: 'filebeat-*', + title: 'filebeat-*', + timeFieldName: '@timestamp', + fields: _.map($scope.sampleDocs, (value, key) => { return {name: key, type: typeof value}; - }); - } + }) + }); + + $scope.isTimeBased = !!$scope.indexPattern.timeFieldName; $scope.$watch('indexPattern.id', function (value) { $scope.indexPattern.title = value; From a47fc05ef79b334fbfe5d0aa30af5ab3bf4ee745 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Wed, 13 Jan 2016 12:03:30 -0500 Subject: [PATCH 06/19] Detect geo_points and dates by looking at the pipeline --- .../data/directives/pattern_review_step.js | 69 +++++++++++++++++-- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index 935314a3788b13..299142f9381148 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -4,16 +4,55 @@ var _ = require('lodash'); var editFieldTypeHTML = require('plugins/kibana/settings/sections/data/partials/_edit_field_type.html'); var testData = { - message: 'src=1.1.1.1 evil=1', - src: '1.1.1.1', - evil: '1', - coordinates: { + message: '11/24/2015 ip=1.1.1.1 bytes=1234', + clientip: '1.1.1.1', + bytes: 1234, + geoip: { lat: 37.3894, lon: 122.0819 }, - '@timestamp': '2015-11-24T00:00:00.000Z' + location: { + lat: 37.3894, + lon: 122.0819 + }, + '@timestamp': '2015-11-24T00:00:00.000Z', + otherdate: '2015-11-24T00:00:00.000Z', + codes: [1, 2, 3, 4] }; +var testPipeline = [ + { + grok: { + match_field: 'message', + match_pattern: 'foo' + } + }, + { + geoip: { + source_field: 'ip' + } + }, + { + geoip: { + source_field: 'ip', + target_field: 'location' + } + }, + { + date: { + match_field: 'initialDate', + match_formats: ['dd/MM/yyyy hh:mm:ss'] + } + }, + { + date: { + match_field: 'initialDate', + match_formats: ['dd/MM/yyyy hh:mm:ss'], + target_field: 'otherdate' + } + } +]; + modules.get('apps/settings') .directive('patternReviewStep', function () { return { @@ -25,16 +64,31 @@ modules.get('apps/settings') }, controller: function ($scope, Private) { $scope.sampleDocs = testData; + $scope.pipeline = testPipeline; + if (_.isUndefined($scope.indexPattern)) { $scope.indexPattern = {}; } + const knownFieldTypes = {}; + $scope.pipeline.forEach(function (processor) { + if (processor.geoip) { + let field = processor.geoip.target_field || 'geoip'; + knownFieldTypes[field] = 'geo_point'; + } + if (processor.date) { + let field = processor.date.target_field || '@timestamp'; + knownFieldTypes[field] = 'date'; + } + }); + _.defaults($scope.indexPattern, { id: 'filebeat-*', title: 'filebeat-*', timeFieldName: '@timestamp', fields: _.map($scope.sampleDocs, (value, key) => { - return {name: key, type: typeof value}; + let type = knownFieldTypes[key] || typeof value; + return {name: key, type: type}; }) }); @@ -51,13 +105,14 @@ modules.get('apps/settings') ]; $scope.rows = _.map($scope.indexPattern.fields, (field) => { + const sampleValue = $scope.sampleDocs[field.name]; return [ field.name, { markup: editFieldTypeHTML, scope: _.assign($scope.$new(), {field: field}) }, - $scope.sampleDocs[field.name] + typeof sampleValue === 'object' ? JSON.stringify(sampleValue) : sampleValue ]; }); } From 3bad81de585275aa418887b376dffe321ec4b8ed Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Wed, 13 Jan 2016 12:07:09 -0500 Subject: [PATCH 07/19] detect the type of array elements --- .../settings/sections/data/directives/pattern_review_step.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index 299142f9381148..df75d5f2d8147d 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -88,6 +88,9 @@ modules.get('apps/settings') timeFieldName: '@timestamp', fields: _.map($scope.sampleDocs, (value, key) => { let type = knownFieldTypes[key] || typeof value; + if (type === 'object' && _.isArray(value) && !_.isEmpty(value)) { + type = typeof value[0]; + } return {name: key, type: type}; }) }); From 9e4362b2456764be2ede6582e104746caed4f98a Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Wed, 13 Jan 2016 14:10:58 -0500 Subject: [PATCH 08/19] Pick default date field and update date field list when user changes type --- .../data/directives/pattern_review_step.html | 6 +++- .../data/directives/pattern_review_step.js | 28 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html index aa4f427dbfb379..e97520f0b0ba09 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.html @@ -8,7 +8,11 @@

Pattern review step

diff --git a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js index df75d5f2d8147d..a7a4f838a38290 100644 --- a/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/data/directives/pattern_review_step.js @@ -53,6 +53,19 @@ var testPipeline = [ } ]; +function pickDefaultTimeFieldName(dateFields) { + if (_.isEmpty(dateFields)) { + return undefined; + } + + let fieldName = dateFields[0]; + if (_.includes(dateFields, '@timestamp')) { + fieldName = '@timestamp'; + } + + return fieldName; +} + modules.get('apps/settings') .directive('patternReviewStep', function () { return { @@ -71,6 +84,7 @@ modules.get('apps/settings') } const knownFieldTypes = {}; + $scope.dateFields = []; $scope.pipeline.forEach(function (processor) { if (processor.geoip) { let field = processor.geoip.target_field || 'geoip'; @@ -79,13 +93,14 @@ modules.get('apps/settings') if (processor.date) { let field = processor.date.target_field || '@timestamp'; knownFieldTypes[field] = 'date'; + $scope.dateFields.push(field); } }); _.defaults($scope.indexPattern, { id: 'filebeat-*', title: 'filebeat-*', - timeFieldName: '@timestamp', + timeFieldName: pickDefaultTimeFieldName($scope.dateFields), fields: _.map($scope.sampleDocs, (value, key) => { let type = knownFieldTypes[key] || typeof value; if (type === 'object' && _.isArray(value) && !_.isEmpty(value)) { @@ -100,6 +115,17 @@ modules.get('apps/settings') $scope.$watch('indexPattern.id', function (value) { $scope.indexPattern.title = value; }); + $scope.$watch('isTimeBased', function (value) { + if (value) { + $scope.indexPattern.timeFieldName = pickDefaultTimeFieldName($scope.dateFields); + } + else { + delete $scope.indexPattern.timeFieldName; + } + }); + $scope.$watch('indexPattern.fields', (fields) => { + $scope.dateFields = _.map(_.filter(fields, {type: 'date'}), 'name'); + }, true); $scope.columns = [ {title: 'Field'}, From 4df54be44c073277fc45c0b6f814ed390a4341b5 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Wed, 13 Jan 2016 14:17:11 -0500 Subject: [PATCH 09/19] lock type if we detect a geo_point --- .../settings/sections/data/partials/_edit_field_type.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html b/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html index eecf3a3f35481f..accc574f4d727b 100644 --- a/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html +++ b/src/plugins/kibana/public/settings/sections/data/partials/_edit_field_type.html @@ -1,4 +1,4 @@ - @@ -7,3 +7,7 @@ + + + geo_point + From 7ae9d3add81cfdd2cb0bb0766aab3ea91de63251 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Mon, 25 Jan 2016 18:47:35 -0500 Subject: [PATCH 10/19] Some styling and layout --- .../indices/directives/pattern_review_step.html | 12 ++++++++---- .../kibana/public/settings/styles/main.less | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/directives/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/indices/directives/pattern_review_step.html index e97520f0b0ba09..d98f2e43ba6e49 100644 --- a/src/plugins/kibana/public/settings/sections/indices/directives/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/indices/directives/pattern_review_step.html @@ -1,12 +1,16 @@ -

Pattern review step

+
+

Review the index pattern

+ Here we'll define how and where to store your parsed events. We've made some intellient guesses for you, but most + fields can be changed iw we got it wrong! +
-
- +
+ -
- + -