Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[indexPatterns/field] remove indexed/analyzed/doc_values awareness #11969

Merged
merged 5 commits into from
May 25, 2017
Merged

[indexPatterns/field] remove indexed/analyzed/doc_values awareness #11969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
23a30a6
[indexPatterns/field] replace mapping awareness with readFromDocValues
spalger May 24, 2017
beaed69
move link to more details to proper location
spalger May 25, 2017
f708d6c
Merge branch 'master' of github.com:elastic/kibana into index-pattern…
spalger May 25, 2017
ecc16c4
Address kjbekkelund's feedback
spalger May 25, 2017
cefd8ad
let the prs speak for themselves
spalger May 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 0 additions & 17 deletions src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,6 @@ app.directive('discoverField', function ($compile) {
const getWarnings = function (field) {
let warnings = [];

if (!field.scripted) {
if (!field.doc_values && field.type !== 'boolean' && !(field.analyzed && field.type === 'string')) {
warnings.push('Doc values are not enabled on this field. This may lead to excess heap consumption when visualizing.');
}

if (field.analyzed && field.type === 'string') {
warnings.push('This is an analyzed string field.' +
' Analyzed strings are highly unique and can use a lot of memory to visualize.' +
' Values such as foo-bar will be broken into foo and bar.');
}

if (!field.indexed && !field.searchable) {
warnings.push('This field is not indexed and might not be usable in visualizations.');
}
}


if (field.scripted) {
warnings.push('Scripted fields can take a long time to execute.');
}
Expand Down
8 changes: 4 additions & 4 deletions src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,21 +62,21 @@ <h5>Available Fields
<form role="form" class="discover-field-details">
<div class="form-group">
<label>
Analyzed
Aggregatable
</label>
<select
ng-options="opt.value as opt.label for opt in filter.boolOpts"
ng-model="filter.vals.analyzed"
ng-model="filter.vals.aggregatable"
class="form-control">
</select>
</div>
<div class="form-group">
<label>
Indexed
Searchable
</label>
<select
ng-options="opt.value as opt.label for opt in filter.boolOpts"
ng-model="filter.vals.indexed"
ng-model="filter.vals.searchable"
class="form-control">
</select>
</div>
Expand Down
19 changes: 10 additions & 9 deletions src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,14 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
const filter = $scope.filter = {
props: [
'type',
'indexed',
'analyzed',
'aggregatable',
'searchable',
'missing',
'name'
],
defaults: {
missing: true
missing: true,
type: 'any'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the benefit of any over null or undefined?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's the value rendered in the UI. This is master:

image

},
boolOpts: [
{ label: 'any', value: undefined },
Expand All @@ -69,16 +70,16 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
return field.display;
},
isFieldFiltered: function (field) {
const matchFilter = (filter.vals.type == null || field.type === filter.vals.type);
const isAnalyzed = (filter.vals.analyzed == null || field.analyzed === filter.vals.analyzed);
const isIndexed = (filter.vals.indexed == null || field.indexed === filter.vals.indexed);
const matchFilter = (filter.vals.type === 'any' || field.type === filter.vals.type);
const isAggregatable = (filter.vals.aggregatable == null || field.aggregatable === filter.vals.aggregatable);
const isSearchable = (filter.vals.searchable == null || field.searchable === filter.vals.searchable);
const scriptedOrMissing = (!filter.vals.missing || field.scripted || field.rowCount > 0);
const matchName = (!filter.vals.name || field.name.indexOf(filter.vals.name) !== -1);

return !field.display
&& matchFilter
&& isAnalyzed
&& isIndexed
&& isAggregatable
&& isSearchable
&& scriptedOrMissing
&& matchName
;
Expand Down Expand Up @@ -146,7 +147,7 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
.commit();

// include undefined so the user can clear the filter
$scope.fieldTypes = _.union([undefined], _.pluck(fields, 'type'));
$scope.fieldTypes = _.union(['any'], _.pluck(fields, 'type'));
});

$scope.increaseFieldCounter = function (fieldName) {
Expand Down
5 changes: 0 additions & 5 deletions ...nagement/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ uiModules.get('apps/management')
{ title: 'format' },
{ title: 'searchable', info: 'These fields can be used in the filter bar' },
{ title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
{ title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
{ title: 'excluded', info: 'Fields that are excluded from _source when it is fetched' },
{ title: 'controls', sortable: false }
];
Expand Down Expand Up @@ -77,10 +76,6 @@ uiModules.get('apps/management')
markup: field.aggregatable ? yesTemplate : noTemplate,
value: field.aggregatable
},
{
markup: field.analyzed ? yesTemplate : noTemplate,
value: field.analyzed
},
{
markup: excluded ? yesTemplate : noTemplate,
value: excluded
Expand Down
8 changes: 7 additions & 1 deletion src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import _ from 'lodash';

import handleESError from '../../../lib/handle_es_error';
import { shouldReadFieldFromDocValues } from './should_read_field_from_doc_values';

export function registerFieldCapabilities(server) {
server.route({
Expand All @@ -19,7 +21,11 @@ export function registerFieldCapabilities(server) {
(res) => {
const fields = _.get(res, 'indices._all.fields', {});
const fieldsFilteredValues = _.mapValues(fields, (value) => {
return _.pick(value, ['searchable', 'aggregatable']);
return {
searchable: value.searchable,
aggregatable: value.aggregatable,
readFromDocValues: shouldReadFieldFromDocValues(value.aggregatable, value.type)
};
});

const retVal = { fields: fieldsFilteredValues };
Expand Down
4 changes: 4 additions & 0 deletions src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// see https://github.com/elastic/kibana/issues/11141 for details
export function shouldReadFieldFromDocValues(aggregatable, esType) {
return aggregatable && esType !== 'text' && !esType.startsWith('_');
}
70 changes: 32 additions & 38 deletions src/fixtures/logstash_fields.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,52 +1,48 @@
import { castEsToKbnFieldTypeName } from '../utils';
import { shouldReadFieldFromDocValues } from '../core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values';

function stubbedLogstashFields() {
return [
// |indexed
// | |analyzed
// | | |aggregatable
// | | | |searchable
// name esType | | | | |metadata
['bytes', 'long', true, true, true, true, { count: 10, docValues: true } ],
['ssl', 'boolean', true, true, true, true, { count: 20 } ],
['@timestamp', 'date', true, true, true, true, { count: 30 } ],
['time', 'date', true, true, true, true, { count: 30 } ],
['@tags', 'keyword', true, true, true, true ],
['utc_time', 'date', true, true, true, true ],
['phpmemory', 'integer', true, true, true, true ],
['ip', 'ip', true, true, true, true ],
['request_body', 'attachment', true, true, true, true ],
['point', 'geo_point', true, true, true, true ],
['area', 'geo_shape', true, true, true, true ],
['hashed', 'murmur3', true, true, false, true ],
['geo.coordinates', 'geo_point', true, true, true, true ],
['extension', 'keyword', true, true, true, true ],
['machine.os', 'text', true, true, true, true ],
['machine.os.raw', 'keyword', true, false, true, true, { docValues: true } ],
['geo.src', 'keyword', true, true, true, true ],
['_id', 'keyword', false, false, true, true ],
['_type', 'keyword', false, false, true, true ],
['_source', 'keyword', false, false, true, true ],
['non-filterable', 'text', false, false, true, false],
['non-sortable', 'text', false, false, false, false],
['custom_user_field', 'conflict', false, false, true, true ],
['script string', 'text', false, false, true, false, { script: '\'i am a string\'' } ],
['script number', 'long', false, false, true, false, { script: '1234' } ],
['script date', 'date', false, false, true, false, { script: '1234', lang: 'painless' } ],
['script murmur3', 'murmur3', false, false, true, false, { script: '1234' } ],
// |aggregatable
// | |searchable
// name esType | | |metadata
['bytes', 'long', true, true, { count: 10 } ],
['ssl', 'boolean', true, true, { count: 20 } ],
['@timestamp', 'date', true, true, { count: 30 } ],
['time', 'date', true, true, { count: 30 } ],
['@tags', 'keyword', true, true ],
['utc_time', 'date', true, true ],
['phpmemory', 'integer', true, true ],
['ip', 'ip', true, true ],
['request_body', 'attachment', true, true ],
['point', 'geo_point', true, true ],
['area', 'geo_shape', true, true ],
['hashed', 'murmur3', false, true ],
['geo.coordinates', 'geo_point', true, true ],
['extension', 'keyword', true, true ],
['machine.os', 'text', true, true ],
['machine.os.raw', 'keyword', true, true ],
['geo.src', 'keyword', true, true ],
['_id', '_id', true, true ],
['_type', '_type', true, true ],
['_source', '_source', true, true ],
['non-filterable', 'text', true, false],
['non-sortable', 'text', false, false],
['custom_user_field', 'conflict', true, true ],
['script string', 'text', true, false, { script: '\'i am a string\'' } ],
['script number', 'long', true, false, { script: '1234' } ],
['script date', 'date', true, false, { script: '1234', lang: 'painless' } ],
['script murmur3', 'murmur3', true, false, { script: '1234' } ],
].map(function (row) {
const [
name,
esType,
indexed,
analyzed,
aggregatable,
searchable,
metadata = {}
] = row;

const {
docValues = false,
count = 0,
script,
lang = script ? 'expression' : undefined,
Expand All @@ -60,9 +56,7 @@ function stubbedLogstashFields() {
return {
name,
type,
doc_values: docValues,
indexed,
analyzed,
readFromDocValues: shouldReadFieldFromDocValues(aggregatable, esType),
aggregatable,
searchable,
count,
Expand Down
20 changes: 16 additions & 4 deletions src/ui/public/agg_types/__tests__/metrics/top_hit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,27 @@ describe('Top hit metric', function () {
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});

it('should request both for the source and doc_values fields', function () {
it('requests both source and docvalues_fields for non-text aggregatable fields', function () {
init({ field: 'bytes' });
expect(aggDsl.top_hits._source).to.be('bytes');
expect(aggDsl.top_hits.docvalue_fields).to.eql([ 'bytes' ]);
});

it('should only request for the source if the field does not have the doc_values property', function () {
init({ field: 'ssl' });
expect(aggDsl.top_hits._source).to.be('ssl');
it('requests just source for aggregatable text fields', function () {
init({ field: 'machine.os' });
expect(aggDsl.top_hits._source).to.be('machine.os');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});

it('requests just source for not-aggregatable text fields', function () {
init({ field: 'non-sortable' });
expect(aggDsl.top_hits._source).to.be('non-sortable');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});

it('requests just source for not-aggregatable, non-text fields', function () {
init({ field: 'hashed' });
expect(aggDsl.top_hits._source).to.be('hashed');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});

Expand Down
19 changes: 0 additions & 19 deletions src/ui/public/agg_types/controls/field.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,6 @@
Field
</label>

<span class="pull-right text-warning hintbox-label" ng-show="aggParam.showAnalyzedWarning && agg.params.field.analyzed"
ng-click="showAnalyzedFieldWarning = !showAnalyzedFieldWarning">
<i class="fa fa-warning"></i> Analyzed Field
</span>
<div class="hintbox" ng-show="showAnalyzedFieldWarning && agg.params.field.analyzed">
<p>
<strong>Careful!</strong> The field selected contains analyzed strings. Analyzed strings are highly unique and can use a lot of memory to visualize. Values such as <i>foo-bar</i> will be broken into <i>foo</i> and <i>bar</i>. See <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html" target="_blank">Mapping Types</a> for more information on setting this field as <i>keyword</i> (not analyzed).
</p>

<p ng-show="indexedFields.byName[agg.params.field.name + '.keyword'].analyzed == false">
<strong>Tip:</strong> <i>{{agg.params.field.name + '.keyword'}}</i> may be a non analyzed version of this field.
</p>

<p ng-show="indexedFields.byName[agg.params.field.name + '.raw'].analyzed == false">
<strong>Tip:</strong> <i>{{agg.params.field.name + '.raw'}}</i> may be a non analyzed version of this field.
</p>
</div>


<ui-select
name="field"
required
Expand Down
3 changes: 1 addition & 2 deletions src/ui/public/agg_types/metrics/top_hit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ export function AggTypesMetricsTopHitProvider(Private) {
{
name: 'field',
onlyAggregatable: false,
showAnalyzedWarning: false,
filterFieldTypes: function (vis, value) {
if (vis.type.name === 'table' || vis.type.name === 'metric') {
return true;
Expand All @@ -45,7 +44,7 @@ export function AggTypesMetricsTopHitProvider(Private) {
}
};
} else {
if (field.doc_values) {
if (field.readFromDocValues) {
output.params.docvalue_fields = [ field.name ];
}
output.params._source = field.name === '_source' ? true : field.name;
Expand Down
2 changes: 0 additions & 2 deletions src/ui/public/agg_types/param_types/field.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ export function AggTypesParamTypesFieldProvider(Private, $filter) {
FieldAggParam.prototype.filterFieldTypes = '*';
// retain only the fields with the aggregatable property if the onlyAggregatable option is true
FieldAggParam.prototype.onlyAggregatable = true;
// show a warning about the field being analyzed
FieldAggParam.prototype.showAnalyzedWarning = true;

/**
* Called to serialize values for saving an aggConfig object
Expand Down
3 changes: 2 additions & 1 deletion src/ui/public/index_patterns/__tests__/_index_pattern.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ describe('index pattern', function () {
sortable: true,
type: 'number',
aggregatable: true,
searchable: false
searchable: false,
readFromDocValues: true
},
{
name: 'script number',
Expand Down
25 changes: 20 additions & 5 deletions src/ui/public/index_patterns/_field.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
const scripted = !!spec.scripted;
const searchable = !!spec.searchable || scripted;
const aggregatable = !!spec.aggregatable || scripted;
const readFromDocValues = !!spec.readFromDocValues && !scripted;
const sortable = spec.name === '_score' || ((indexed || aggregatable) && type.sortable);
const filterable = spec.name === '_id' || scripted || ((indexed || searchable) && type.filterable);
const visualizable = aggregatable;
Expand All @@ -55,14 +56,10 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
obj.writ('script', scripted ? spec.script : null);
obj.writ('lang', scripted ? (spec.lang || 'painless') : null);

// mapping info
obj.fact('indexed', indexed);
obj.fact('analyzed', !!spec.analyzed);
obj.fact('doc_values', !!spec.doc_values);

// stats
obj.fact('searchable', searchable);
obj.fact('aggregatable', aggregatable);
obj.fact('readFromDocValues', readFromDocValues);

// usage flags, read-only and won't be saved
obj.comp('format', format);
Expand All @@ -81,6 +78,24 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
return obj.create();
}

Object.defineProperties(Field.prototype, {
indexed: {
get() {
throw new Error('field.indexed has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
analyzed: {
get() {
throw new Error('field.analyzed has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
doc_values: {
get() {
throw new Error('field.doc_values has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
});

Field.prototype.routes = {
edit: '/management/kibana/indices/{{indexPattern.id}}/field/{{name}}'
};
Expand Down