Skip to content

Commit

Permalink
Core: Enable doc values by default, when appropriate
Browse files Browse the repository at this point in the history 
Doc values significantly reduced heap usage, which results in faster
GCs. This change makes the default for doc values dynamic: any
field that is indexed but not analyzed now has doc values. This only
affects fields on indexes created with 2.0+.

closes elastic#8312
closes elastic#10209
  • Loading branch information
@rjernst
rjernst committed Mar 27, 2015
1 parent 3f459f6 commit e0334dc
Show file tree
Hide file tree
Showing 38 changed files with 378 additions and 263 deletions.
48 changes: 32 additions & 16 deletions src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View file
Expand Up @@ -90,7 +90,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {

public static class Defaults {
public static final FieldType FIELD_TYPE = new FieldType();
public static final boolean DOC_VALUES = false;
public static final boolean PRE_2X_DOC_VALUES = false;

static {
FIELD_TYPE.setTokenized(true);
Expand Down Expand Up @@ -302,7 +302,7 @@ protected String buildFullName(BuilderContext context) {
protected FieldDataType fieldDataType;
protected final MultiFields multiFields;
protected CopyTo copyTo;
protected final boolean writePre2xSettings;
protected final boolean indexCreatedBefore2x;

protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, Boolean docValues, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, SimilarityProvider similarity,
Expand All @@ -319,8 +319,10 @@ protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, Boo
this.boost = boost;
this.fieldType = fieldType;
this.fieldType.freeze();

if (indexAnalyzer == null && this.fieldType.tokenized() == false && this.fieldType.indexOptions() != IndexOptions.NONE) {
this.indexCreatedBefore2x = Version.indexCreated(indexSettings).before(Version.V_2_0_0);

boolean indexedNotAnalyzed = this.fieldType.tokenized() == false && this.fieldType.indexOptions() != IndexOptions.NONE;
if (indexAnalyzer == null && indexedNotAnalyzed) {
this.indexAnalyzer = this.searchAnalyzer = Lucene.KEYWORD_ANALYZER;
} else {
this.indexAnalyzer = indexAnalyzer;
Expand All @@ -339,16 +341,22 @@ protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, Boo
ImmutableSettings.builder().put(defaultFieldDataType().getSettings()).put(fieldDataSettings)
);
}

if (docValues != null) {
// explicitly set
this.docValues = docValues;
} else if (fieldDataType == null) {
this.docValues = false;
} else if (fieldDataType != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldDataType.getFormat(indexSettings))) {
// convoluted way to enable doc values, should be removed in the future
this.docValues = true;
} else if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0)) {
// 2.0+ index, default to true when appropriate
this.docValues = defaultDocValues();
} else {
this.docValues = FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldDataType.getFormat(indexSettings));
}
// old default, disable
this.docValues = false;
}
this.multiFields = multiFields;
this.copyTo = copyTo;
this.writePre2xSettings = Version.indexCreated(indexSettings).before(Version.V_2_0_0);
}

@Nullable
Expand All @@ -360,6 +368,19 @@ protected String defaultPostingFormat() {
protected String defaultDocValuesFormat() {
return null;
}

protected boolean defaultDocValues() {
if (indexCreatedBefore2x) {
return Defaults.PRE_2X_DOC_VALUES;
} else {
return fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE;
}
}

@Override
public final boolean hasDocValues() {
return docValues;
}

@Override
public String name() {
Expand Down Expand Up @@ -683,7 +704,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {

builder.field("type", contentType());
if (writePre2xSettings && (includeDefaults || !names.name().equals(names.indexNameClean()))) {
if (indexCreatedBefore2x && (includeDefaults || !names.name().equals(names.indexNameClean()))) {
builder.field("index_name", names.indexNameClean());
}

Expand All @@ -701,7 +722,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
if (includeDefaults || fieldType.stored() != defaultFieldType.stored()) {
builder.field("store", fieldType.stored());
}
if (includeDefaults || hasDocValues() != Defaults.DOC_VALUES) {
if (includeDefaults || hasDocValues() != defaultDocValues()) {
builder.field(TypeParsers.DOC_VALUES, docValues);
}
if (includeDefaults || fieldType.storeTermVectors() != defaultFieldType.storeTermVectors()) {
Expand Down Expand Up @@ -824,11 +845,6 @@ public boolean supportsNullValue() {
return true;
}

@Override
public boolean hasDocValues() {
return docValues;
}

@Override
public Loading normsLoading(Loading defaultLoading) {
return normsLoading == null ? defaultLoading : normsLoading;
Expand Down
7 changes: 1 addition & 6 deletions src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
View file
Expand Up @@ -131,7 +131,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue,
SimilarityProvider similarity, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(names, boost, fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields, copyTo);
super(names, boost, fieldType, false, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields, copyTo);
this.nullValue = nullValue;
}

Expand Down Expand Up @@ -256,9 +256,4 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field("null_value", nullValue);
}
}

@Override
public boolean hasDocValues() {
return false;
}
}
7 changes: 1 addition & 6 deletions src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View file
Expand Up @@ -240,7 +240,7 @@ private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name)
// with older postings formats such as Elasticsearch090
public CompletionFieldMapper(Names names, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormat wrappedPostingsFormat, SimilarityProvider similarity, boolean payloads,
boolean preserveSeparators, boolean preservePositionIncrements, int maxInputLength, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, SortedMap<String, ContextMapping> contextMappings) {
super(names, 1.0f, Defaults.FIELD_TYPE, null, indexAnalyzer, searchAnalyzer, similarity, null, null, indexSettings, multiFields, copyTo);
super(names, 1.0f, Defaults.FIELD_TYPE, false, indexAnalyzer, searchAnalyzer, similarity, null, null, indexSettings, multiFields, copyTo);
analyzingSuggestLookupProvider = new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, payloads);
if (wrappedPostingsFormat == null) {
// delayed until postingsFormat() is called
Expand Down Expand Up @@ -494,11 +494,6 @@ public boolean isSortable() {
return false;
}

@Override
public boolean hasDocValues() {
return false;
}

@Override
public boolean supportsNullValue() {
return false;
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
View file
Expand Up @@ -197,8 +197,8 @@ public GeoPointFieldMapper build(BuilderContext context) {
latMapperBuilder.precisionStep(precisionStep);
lonMapperBuilder.precisionStep(precisionStep);
}
latMapper = (DoubleFieldMapper) latMapperBuilder.includeInAll(false).store(fieldType.stored()).build(context);
lonMapper = (DoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(fieldType.stored()).build(context);
latMapper = (DoubleFieldMapper) latMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
lonMapper = (DoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
}
StringFieldMapper geohashMapper = null;
if (enableGeoHash) {
Expand Down Expand Up @@ -470,6 +470,11 @@ public FieldType defaultFieldType() {
public FieldDataType defaultFieldDataType() {
return new FieldDataType("geo_point");
}

@Override
protected boolean defaultDocValues() {
return false;
}

public DoubleFieldMapper latMapper() {
return latMapper;
Expand Down
7 changes: 1 addition & 6 deletions src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
View file
Expand Up @@ -215,7 +215,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext

public GeoShapeFieldMapper(FieldMapper.Names names, SpatialPrefixTree tree, String defaultStrategyName, double distanceErrorPct,
Orientation shapeOrientation, FieldType fieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(names, 1, fieldType, null, null, null, null, null, null, indexSettings, multiFields, copyTo);
super(names, 1, fieldType, false, null, null, null, null, null, indexSettings, multiFields, copyTo);
this.recursiveStrategy = new RecursivePrefixTreeStrategy(tree, names.indexName());
this.recursiveStrategy.setDistErrPct(distanceErrorPct);
this.termStrategy = new TermQueryPrefixTreeStrategy(tree, names.indexName());
Expand All @@ -234,11 +234,6 @@ public FieldDataType defaultFieldDataType() {
return null;
}

@Override
public boolean hasDocValues() {
return false;
}

@Override
public void parse(ParseContext context) throws IOException {
try {
Expand Down
26 changes: 22 additions & 4 deletions src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View file
Expand Up @@ -26,6 +26,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
Expand All @@ -34,6 +35,7 @@
import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
Expand All @@ -55,6 +57,7 @@
import java.util.Map;

import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
import static org.elasticsearch.index.mapper.MapperBuilders.all;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;

Expand Down Expand Up @@ -125,6 +128,24 @@ public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
AllFieldMapper.Builder builder = all();

// parseField below will happily parse the doc_values setting, but it is then never passed to
// the AllFieldMapper ctor in the builder since it is not valid. Here we validate
// the doc values settings (old and new) are rejected
Object docValues = node.get("doc_values");
if (docValues != null && nodeBooleanValue(docValues)) {
throw new MapperParsingException("Field [" + name + "] is always tokenized and cannot have doc values");
}
// convoluted way of specifying doc values
Object fielddata = node.get("fielddata");
if (fielddata != null) {
Map<String, Object> fielddataMap = nodeMapValue(fielddata, "fielddata");
Object format = fielddataMap.get("format");
if ("doc_values".equals(format)) {
throw new MapperParsingException("Field [" + name + "] is always tokenized and cannot have doc values");
}
}

parseField(builder, builder.name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
Expand Down Expand Up @@ -158,11 +179,8 @@ public AllFieldMapper(Settings indexSettings) {
protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
EnabledAttributeMapper enabled, boolean autoBoost, SimilarityProvider similarity, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, name, name, name), 1.0f, fieldType, null, indexAnalyzer, searchAnalyzer,
super(new Names(name, name, name, name), 1.0f, fieldType, false, indexAnalyzer, searchAnalyzer,
similarity, normsLoading, fieldDataSettings, indexSettings);
if (hasDocValues()) {
throw new MapperParsingException("Field [" + names.fullName() + "] is always tokenized and cannot have doc values");
}
this.enabledState = enabled;
this.autoBoost = autoBoost;

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
View file
Expand Up @@ -268,7 +268,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeDefaults || enabledState != Defaults.ENABLED_STATE) {
builder.field("enabled", enabledState.enabled);
}
if (writePre2xSettings && (includeDefaults || fieldType().equals(Defaults.FIELD_TYPE) == false)) {
if (indexCreatedBefore2x && (includeDefaults || fieldType().equals(Defaults.FIELD_TYPE) == false)) {
super.doXContentBody(builder, includeDefaults, params);
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
View file
Expand Up @@ -349,7 +349,7 @@ protected String contentType() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (writePre2xSettings == false) {
if (indexCreatedBefore2x == false) {
return builder;
}
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
Expand Down
18 changes: 6 additions & 12 deletions src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
View file
Expand Up @@ -27,7 +27,6 @@
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.fielddata.FieldDataType;
Expand Down Expand Up @@ -90,7 +89,7 @@ public Builder enabled(EnabledAttributeMapper enabledState) {

@Override
public IndexFieldMapper build(BuilderContext context) {
return new IndexFieldMapper(name, indexName, boost, fieldType, docValues, enabledState, fieldDataSettings, context.indexSettings());
return new IndexFieldMapper(name, indexName, boost, fieldType, enabledState, fieldDataSettings, context.indexSettings());
}
}

Expand Down Expand Up @@ -119,12 +118,12 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
private EnabledAttributeMapper enabledState;

public IndexFieldMapper(Settings indexSettings) {
this(Defaults.NAME, Defaults.NAME, Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED_STATE, null, indexSettings);
this(Defaults.NAME, Defaults.NAME, Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), Defaults.ENABLED_STATE, null, indexSettings);
}

public IndexFieldMapper(String name, String indexName, float boost, FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState,
public IndexFieldMapper(String name, String indexName, float boost, FieldType fieldType, EnabledAttributeMapper enabledState,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, indexName, indexName, name), boost, fieldType, docValues, Lucene.KEYWORD_ANALYZER,
super(new Names(name, indexName, indexName, name), boost, fieldType, false, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, null, null, fieldDataSettings, indexSettings);
this.enabledState = enabledState;
}
Expand All @@ -143,11 +142,6 @@ public FieldDataType defaultFieldDataType() {
return new FieldDataType(IndexFieldMapper.NAME);
}

@Override
public boolean hasDocValues() {
return false;
}

public String value(Document document) {
Field field = (Field) document.getField(names.indexName());
return field == null ? null : value(field);
Expand Down Expand Up @@ -203,14 +197,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder;
}
builder.startObject(CONTENT_TYPE);
if (writePre2xSettings && (includeDefaults || fieldType().stored() != Defaults.FIELD_TYPE.stored())) {
if (indexCreatedBefore2x && (includeDefaults || fieldType().stored() != Defaults.FIELD_TYPE.stored())) {
builder.field("store", fieldType().stored());
}
if (includeDefaults || enabledState != Defaults.ENABLED_STATE) {
builder.field("enabled", enabledState.enabled);
}

if (writePre2xSettings) {
if (indexCreatedBefore2x) {
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {
Expand Down
7 changes: 1 addition & 6 deletions src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View file
Expand Up @@ -151,7 +151,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
private final BytesRef typeAsBytes;

protected ParentFieldMapper(String name, String indexName, String type, @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), null,
super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), false,
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, null, fieldDataSettings, indexSettings);
this.type = type;
this.typeAsBytes = type == null ? null : new BytesRef(type);
Expand All @@ -176,11 +176,6 @@ public FieldDataType defaultFieldDataType() {
return new FieldDataType("_parent", settingsBuilder().put(Loading.KEY, Loading.EAGER_VALUE));
}

@Override
public boolean hasDocValues() {
return false;
}

@Override
public void preParse(ParseContext context) throws IOException {
}
Expand Down

0 comments on commit e0334dc

Please sign in to comment.