Permalink
Browse files

Added a global ignore_malformed index setting. #2220 Also extended th…

…e ignore_malformed support to TTL, Ip and timestamp field types.
  • Loading branch information...
1 parent 888b7cc commit cd0e1226e1788c376ba1030a241d7d83d36a7cf1 @martijnvg martijnvg committed with kimchy Aug 31, 2012
@@ -76,7 +76,7 @@ public Builder nullValue(byte nullValue) {
@Override
public ByteFieldMapper build(BuilderContext context) {
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
- precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed);
+ precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -105,7 +105,7 @@ public DateFieldMapper build(BuilderContext context) {
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
- timeUnit, parseUpperInclusive, ignoreMalformed);
+ timeUnit, parseUpperInclusive, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -77,7 +77,7 @@ public Builder nullValue(double nullValue) {
public DoubleFieldMapper build(BuilderContext context) {
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
- ignoreMalformed);
+ ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -78,7 +78,7 @@ public Builder nullValue(float nullValue) {
public FloatFieldMapper build(BuilderContext context) {
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
- ignoreMalformed);
+ ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -78,7 +78,7 @@ public Builder nullValue(int nullValue) {
public IntegerFieldMapper build(BuilderContext context) {
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions,
- nullValue, ignoreMalformed);
+ nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -78,7 +78,7 @@ public Builder nullValue(long nullValue) {
public LongFieldMapper build(BuilderContext context) {
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
- ignoreMalformed);
+ ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -60,7 +60,7 @@
protected String fuzzyFactor = Defaults.FUZZY_FACTOR;
- protected boolean ignoreMalformed = Defaults.IGNORE_MALFORMED;
+ private Boolean ignoreMalformed;
public Builder(String name) {
super(name);
@@ -104,6 +104,15 @@ public T ignoreMalformed(boolean ignoreMalformed) {
return builder;
}
+ protected boolean ignoreMalformed(BuilderContext context) {
+ if (ignoreMalformed != null) {
+ return ignoreMalformed;
+ }
+ if (context.indexSettings() != null) {
+ return context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED);
+ }
+ return Defaults.IGNORE_MALFORMED;
+ }
}
protected int precisionStep;
@@ -78,7 +78,7 @@ public Builder nullValue(short nullValue) {
public ShortFieldMapper build(BuilderContext context) {
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
- ignoreMalformed);
+ ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -75,7 +75,7 @@ public Builder defaultTTL(long defaultTTL) {
@Override
public TTLFieldMapper build(BuilderContext context) {
- return new TTLFieldMapper(store, index, enabled, defaultTTL);
+ return new TTLFieldMapper(store, index, enabled, defaultTTL, ignoreMalformed(context));
}
}
@@ -104,13 +104,13 @@ public TTLFieldMapper build(BuilderContext context) {
private long defaultTTL;
public TTLFieldMapper() {
- this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT);
+ this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED);
}
- protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL) {
+ protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL, boolean ignoreMalformed) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
- Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED);
+ Defaults.NULL_VALUE, ignoreMalformed);
this.enabled = enabled;
this.defaultTTL = defaultTTL;
}
@@ -89,7 +89,7 @@ public TimestampFieldMapper build(BuilderContext context) {
if (context.indexSettings() != null) {
parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
}
- return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive);
+ return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive, ignoreMalformed(context));
}
}
@@ -119,14 +119,15 @@ public TimestampFieldMapper build(BuilderContext context) {
private final String path;
public TimestampFieldMapper() {
- this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE);
+ this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE, Defaults.IGNORE_MALFORMED);
}
- protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive) {
+ protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path,
+ FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, boolean ignoreMalformed) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
- parseUpperInclusive, Defaults.IGNORE_MALFORMED);
+ parseUpperInclusive, ignoreMalformed);
this.enabled = enabled;
this.path = path;
}
@@ -105,7 +105,7 @@ public Builder nullValue(String nullValue) {
@Override
public IpFieldMapper build(BuilderContext context) {
IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
- precisionStep, index, store, boost, omitNorms, indexOptions, nullValue);
+ precisionStep, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@@ -132,9 +132,9 @@ public IpFieldMapper build(BuilderContext context) {
protected IpFieldMapper(Names names, int precisionStep,
Field.Index index, Field.Store store,
float boost, boolean omitNorms, IndexOptions indexOptions,
- String nullValue) {
+ String nullValue, boolean ignoreMalformed) {
super(names, precisionStep, null, index, store, boost, omitNorms, indexOptions,
- false, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
+ ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)));
this.nullValue = nullValue;
}
@@ -22,6 +22,7 @@
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.EnvironmentModule;
@@ -44,6 +45,10 @@ public static DocumentMapperParser newParser() {
return new DocumentMapperParser(new Index("test"), newAnalysisService());
}
+ public static DocumentMapperParser newParser(Settings indexSettings) {
+ return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService());
+ }
+
public static MapperService newMapperService() {
return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService());
}
@@ -19,14 +19,17 @@
package org.elasticsearch.test.unit.index.mapper.date;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
@@ -97,4 +100,68 @@ public void testDateDetection() throws Exception {
assertThat(doc.rootDoc().get("date_field"), nullValue());
assertThat(doc.rootDoc().get("date_field_x"), equalTo("2010-01-01"));
}
+
+ @Test
+ public void testIgnoreMalformedOption() throws Exception {
+ String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+ .startObject("properties")
+ .startObject("field1").field("type", "date").field("ignore_malformed", true).endObject()
+ .startObject("field2").field("type", "date").field("ignore_malformed", false).endObject()
+ .startObject("field3").field("type", "date").endObject()
+ .endObject()
+ .endObject().endObject().string();
+
+ DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
+
+ ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field1", "a")
+ .field("field2", "2010-01-01")
+ .endObject()
+ .bytes());
+ assertThat(doc.rootDoc().getFieldable("field1"), nullValue());
+ assertThat(doc.rootDoc().getFieldable("field2"), notNullValue());
+
+ try {
+ defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field2", "a")
+ .endObject()
+ .bytes());
+ } catch (MapperParsingException e) {
+ assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+ }
+
+ // Verify that the default is false
+ try {
+ defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field3", "a")
+ .endObject()
+ .bytes());
+ } catch (MapperParsingException e) {
+ assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+ }
+
+ // Unless the global ignore_malformed option is set to true
+ Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
+ defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
+ doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field3", "a")
+ .endObject()
+ .bytes());
+ assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
+
+ // This should still throw an exception, since field2 is specifically set to ignore_malformed=false
+ try {
+ defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field2", "a")
+ .endObject()
+ .bytes());
+ } catch (MapperParsingException e) {
+ assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+ }
+ }
}
@@ -19,6 +19,7 @@
package org.elasticsearch.test.unit.index.mapper.numeric;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
@@ -30,6 +31,7 @@
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
@@ -79,7 +81,7 @@ public void testNumericDetectionDefault() throws Exception {
assertThat(mapper, instanceOf(StringFieldMapper.class));
}
- public void testIgnoreMalformedEnabled() throws Exception {
+ public void testIgnoreMalformedOption() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "integer").field("ignore_malformed", true).endObject()
@@ -119,6 +121,27 @@ public void testIgnoreMalformedEnabled() throws Exception {
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
+
+ // Unless the global ignore_malformed option is set to true
+ Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
+ defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
+ doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field3", "a")
+ .endObject()
+ .bytes());
+ assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
+
+ // This should still throw an exception, since field2 is specifically set to ignore_malformed=false
+ try {
+ defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field2", "a")
+ .endObject()
+ .bytes());
+ } catch (MapperParsingException e) {
+ assertThat(e.getCause(), instanceOf(NumberFormatException.class));
+ }
}
}

0 comments on commit cd0e122

Please sign in to comment.