Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Completely move doc values and fielddata settings to field types #12014

Merged
merged 3 commits into from Jul 6, 2015
Merged

Completely move doc values and fielddata settings to field types #12014

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e83ae64
Mappings: Completely move doc values and fielddata settings to field …
rjernst Jul 3, 2015
91e3cdb
Addressed PR comments
rjernst Jul 6, 2015
184b1cb
Fix fake field mapper to not pass null for defaultfieldtype
rjernst Jul 6, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 4 additions & 1 deletion core/src/main/java/org/elasticsearch/common/settings/Settings.java
View file
Open in desktop
Expand Up @@ -22,6 +22,7 @@
import com.google.common.base.Charsets;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -81,7 +82,9 @@ public static boolean getSettingsRequireUnits() {
private transient ClassLoader classLoader;

Settings(Map<String, String> settings, ClassLoader classLoader) {
this.settings = ImmutableMap.copyOf(settings);
// we use a sorted map for consistent serialization when using getAsMap()
// TODO: use Collections.unmodifiableMap with a TreeMap
this.settings = ImmutableSortedMap.copyOf(settings);
Map<String, String> forcedUnderscoreSettings = null;
for (Map.Entry<String, String> entry : settings.entrySet()) {
String toUnderscoreCase = Strings.toUnderscoreCase(entry.getKey());
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
View file
Open in desktop
Expand Up @@ -37,6 +37,7 @@
import org.elasticsearch.action.fieldstats.FieldStats;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
Expand Down Expand Up @@ -189,6 +190,7 @@ public MappedFieldType() {
setOmitNorms(false);
setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
setBoost(1.0f);
fieldDataType = new FieldDataType(typeName());
}

public abstract MappedFieldType clone();
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
View file
Open in desktop
Expand Up @@ -38,8 +38,8 @@ public Builder(String name, MappedFieldType fieldType) {
}
}

protected MetadataFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues, @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(simpleName, fieldType, docValues, fieldDataSettings, indexSettings, MultiFields.empty(), null);
protected MetadataFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) {
super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), null);
}

/**
Expand Down
100 changes: 37 additions & 63 deletions core/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View file
Open in desktop
Expand Up @@ -70,12 +70,13 @@ public static class Defaults {
public abstract static class Builder<T extends Builder, Y extends AbstractFieldMapper> extends Mapper.Builder<T, Y> {

protected final MappedFieldType fieldType;
protected final MappedFieldType defaultFieldType;
private final IndexOptions defaultOptions;
protected Boolean docValues;
protected boolean omitNormsSet = false;
protected String indexName;
protected Boolean includeInAll;
protected boolean indexOptionsSet = false;
protected boolean docValuesSet = false;
@Nullable
protected Settings fieldDataSettings;
protected final MultiFields.Builder multiFieldsBuilder;
Expand All @@ -84,6 +85,7 @@ public abstract static class Builder<T extends Builder, Y extends AbstractFieldM
protected Builder(String name, MappedFieldType fieldType) {
super(name);
this.fieldType = fieldType.clone();
this.defaultFieldType = fieldType.clone();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we freeze() it in build()?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, good catch!

this.defaultOptions = fieldType.indexOptions(); // we have to store it the fieldType is mutable
multiFieldsBuilder = new MultiFields.Builder();
}
Expand Down Expand Up @@ -116,7 +118,8 @@ public T store(boolean store) {
}

public T docValues(boolean docValues) {
this.docValues = docValues;
this.fieldType.setHasDocValues(docValues);
this.docValuesSet = true;
return builder;
}

Expand Down Expand Up @@ -253,76 +256,54 @@ protected String buildFullName(BuilderContext context) {

protected void setupFieldType(BuilderContext context) {
fieldType.setNames(buildNames(context));
if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) {
fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
if (fieldDataSettings != null) {
Settings settings = Settings.builder().put(fieldType.fieldDataType().getSettings()).put(fieldDataSettings).build();
fieldType.setFieldDataType(new FieldDataType(fieldType.fieldDataType().getType(), settings));
}
boolean defaultDocValues = false; // pre 2.0
if (context.indexCreatedVersion().onOrAfter(Version.V_2_0_0)) {
defaultDocValues = fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE;
}
// backcompat for "fielddata: format: docvalues" for now...
boolean fieldDataDocValues = fieldType.fieldDataType() != null
&& FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldType.fieldDataType().getFormat(context.indexSettings()));
if (fieldDataDocValues && docValuesSet && fieldType.hasDocValues() == false) {
// this forces the doc_values setting to be written, so fielddata does not mask the original setting
defaultDocValues = true;
}
defaultFieldType.setHasDocValues(defaultDocValues);
if (docValuesSet == false) {
fieldType.setHasDocValues(defaultDocValues || fieldDataDocValues);
}
}
}

protected MappedFieldTypeReference fieldTypeRef;
protected final boolean hasDefaultDocValues;
protected Settings customFieldDataSettings;
protected final MappedFieldType defaultFieldType;
protected final MultiFields multiFields;
protected CopyTo copyTo;
protected final boolean indexCreatedBefore2x;

protected AbstractFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues, @Nullable Settings fieldDataSettings, Settings indexSettings) {
this(simpleName, fieldType, docValues, fieldDataSettings, indexSettings, MultiFields.empty(), null);
}

protected AbstractFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
protected AbstractFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName);
assert indexSettings != null;
this.indexCreatedBefore2x = Version.indexCreated(indexSettings).before(Version.V_2_0_0);
this.customFieldDataSettings = fieldDataSettings;
FieldDataType fieldDataType;
if (fieldDataSettings == null) {
fieldDataType = defaultFieldDataType();
} else {
// create a new field data type, with the default settings as well as the "new ones"
fieldDataType = new FieldDataType(defaultFieldDataType().getType(),
Settings.builder().put(defaultFieldDataType().getSettings()).put(fieldDataSettings)
);
}

// TODO: hasDocValues should just be set directly on the field type by callers of this ctor, but
// then we need to eliminate defaultDocValues() (only needed by geo, which needs to be fixed with passing
// doc values setting down to lat/lon) and get rid of specifying doc values in fielddata (which
// complicates whether we can just compare to the default value to know whether to write the setting)
if (docValues == null && fieldDataType != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldDataType.getFormat(indexSettings))) {
docValues = true;
}
hasDefaultDocValues = docValues == null;

this.fieldTypeRef = new MappedFieldTypeReference(fieldType); // must init first so defaultDocValues() can be called
fieldType = fieldType.clone();
if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) {
fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
fieldType.setHasDocValues(docValues == null ? defaultDocValues() : docValues);
fieldType.setFieldDataType(fieldDataType);
fieldType.freeze();
this.fieldTypeRef.set(fieldType); // now reset ref once extra settings have been initialized

this.fieldTypeRef = new MappedFieldTypeReference(fieldType); // the reference ctor freezes the field type
defaultFieldType.freeze();
this.defaultFieldType = defaultFieldType;
this.multiFields = multiFields;
this.copyTo = copyTo;
}

protected boolean defaultDocValues() {
if (indexCreatedBefore2x) {
return false;
} else {
return fieldType().tokenized() == false && fieldType().indexOptions() != IndexOptions.NONE;
}
}

@Override
public String name() {
return fieldType().names().fullName();
}

public abstract MappedFieldType defaultFieldType();

public abstract FieldDataType defaultFieldDataType();

@Override
public MappedFieldType fieldType() {
return fieldTypeRef.get();
Expand Down Expand Up @@ -417,7 +398,6 @@ public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMapping
MappedFieldType fieldType = fieldMergeWith.fieldType().clone();
fieldType.freeze();
fieldTypeRef.set(fieldType);
this.customFieldDataSettings = fieldMergeWith.customFieldDataSettings;
this.copyTo = fieldMergeWith.copyTo;
}
}
Expand All @@ -441,7 +421,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field("boost", fieldType().boost());
}

FieldType defaultFieldType = defaultFieldType();
boolean indexed = fieldType().indexOptions() != IndexOptions.NONE;
boolean defaultIndexed = defaultFieldType.indexOptions() != IndexOptions.NONE;
if (includeDefaults || indexed != defaultIndexed ||
Expand Down Expand Up @@ -477,13 +456,8 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field("similarity", SimilarityLookupService.DEFAULT_SIMILARITY);
}

TreeMap<String, Object> orderedFielddataSettings = new TreeMap<>();
if (hasCustomFieldDataSettings()) {
orderedFielddataSettings.putAll(customFieldDataSettings.getAsMap());
builder.field("fielddata", orderedFielddataSettings);
} else if (includeDefaults) {
orderedFielddataSettings.putAll(fieldType().fieldDataType().getSettings().getAsMap());
builder.field("fielddata", orderedFielddataSettings);
if (includeDefaults || hasCustomFieldDataSettings()) {
builder.field("fielddata", fieldType().fieldDataType().getSettings().getAsMap());
}
multiFields.toXContent(builder, params);

Expand All @@ -506,7 +480,7 @@ protected void doXContentAnalyzers(XContentBuilder builder, boolean includeDefau
}

protected void doXContentDocValues(XContentBuilder builder, boolean includeDefaults) throws IOException {
if (includeDefaults || hasDefaultDocValues == false) {
if (includeDefaults || defaultFieldType.hasDocValues() != fieldType().hasDocValues()) {
builder.field(DOC_VALUES, fieldType().hasDocValues());
}
}
Expand Down Expand Up @@ -559,7 +533,7 @@ protected static String indexTokenizeOptionToString(boolean indexed, boolean tok
}

protected boolean hasCustomFieldDataSettings() {
return customFieldDataSettings != null && customFieldDataSettings.equals(Settings.EMPTY) == false;
return fieldType().fieldDataType() != null && fieldType().fieldDataType().equals(defaultFieldType.fieldDataType()) == false;
}

protected abstract String contentType();
Expand Down
20 changes: 5 additions & 15 deletions core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View file
Open in desktop
Expand Up @@ -81,8 +81,8 @@ public Builder(String name) {
public BinaryFieldMapper build(BuilderContext context) {
setupFieldType(context);
((BinaryFieldType)fieldType).setTryUncompressing(context.indexCreatedVersion().before(Version.V_2_0_0));
return new BinaryFieldMapper(name, fieldType, docValues,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
return new BinaryFieldMapper(name, fieldType, defaultFieldType,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
}

Expand Down Expand Up @@ -181,19 +181,9 @@ public Object valueForSearch(Object value) {
}
}

protected BinaryFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues,
@Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, docValues, fieldDataSettings, indexSettings, multiFields, copyTo);
}

@Override
public MappedFieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
}

@Override
public FieldDataType defaultFieldDataType() {
return new FieldDataType("binary");
protected BinaryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
}

@Override
Expand Down
21 changes: 5 additions & 16 deletions core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
View file
Open in desktop
Expand Up @@ -90,8 +90,8 @@ public Builder tokenized(boolean tokenized) {
@Override
public BooleanFieldMapper build(BuilderContext context) {
setupFieldType(context);
return new BooleanFieldMapper(name, fieldType, docValues,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
return new BooleanFieldMapper(name, fieldType, defaultFieldType,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
}

Expand Down Expand Up @@ -194,27 +194,16 @@ public boolean useTermQueryWithQueryString() {
}
}

protected BooleanFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues,
@Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, docValues, fieldDataSettings, indexSettings, multiFields, copyTo);
protected BooleanFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
}

@Override
public BooleanFieldType fieldType() {
return (BooleanFieldType) super.fieldType();
}

@Override
public MappedFieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
}

@Override
public FieldDataType defaultFieldDataType() {
// TODO have a special boolean type?
return new FieldDataType(CONTENT_TYPE);
}

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) {
Expand Down
20 changes: 5 additions & 15 deletions core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View file
Open in desktop
Expand Up @@ -81,8 +81,8 @@ public Builder(String name) {
@Override
public ByteFieldMapper build(BuilderContext context) {
setupFieldType(context);
ByteFieldMapper fieldMapper = new ByteFieldMapper(name, fieldType, docValues, ignoreMalformed(context),
coerce(context), fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
ByteFieldMapper fieldMapper = new ByteFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context),
coerce(context), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
Expand Down Expand Up @@ -193,27 +193,17 @@ public FieldStats stats(Terms terms, int maxDoc) throws IOException {
}
}

protected ByteFieldMapper(String simpleName, MappedFieldType fieldType, Boolean docValues,
protected ByteFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
@Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, docValues, ignoreMalformed, coerce, fieldDataSettings, indexSettings, multiFields, copyTo);
Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, indexSettings, multiFields, copyTo);
}

@Override
public ByteFieldType fieldType() {
return (ByteFieldType) super.fieldType();
}

@Override
public MappedFieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
}

@Override
public FieldDataType defaultFieldDataType() {
return new FieldDataType("byte");
}

private static byte parseValue(Object value) {
if (value instanceof Number) {
return ((Number) value).byteValue();
Expand Down
16 changes: 4 additions & 12 deletions core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View file
Open in desktop
Expand Up @@ -226,7 +226,9 @@ public static final class CompletionFieldType extends MappedFieldType {
private AnalyzingCompletionLookupProvider analyzingSuggestLookupProvider;
private SortedMap<String, ContextMapping> contextMapping = ContextMapping.EMPTY_MAPPING;

public CompletionFieldType() {}
public CompletionFieldType() {
setFieldDataType(null);
}

protected CompletionFieldType(CompletionFieldType ref) {
super(ref);
Expand Down Expand Up @@ -312,7 +314,7 @@ public boolean isSortable() {
private int maxInputLength;

public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, int maxInputLength, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, false, null, indexSettings, multiFields, copyTo);
super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
this.maxInputLength = maxInputLength;
}

Expand Down Expand Up @@ -538,16 +540,6 @@ protected String contentType() {
return CONTENT_TYPE;
}

@Override
public MappedFieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
}

@Override
public FieldDataType defaultFieldDataType() {
return null;
}

public boolean isStoringPayloads() {
return fieldType().analyzingSuggestLookupProvider.hasPayloads();
}
Expand Down