Skip to content

Commit

Permalink
Simulate the entire toXContent instead of special caseing
Browse files Browse the repository at this point in the history 
Today we try to detect if we need to generate the mapping or not in
the all mapper. This is error prone since it misses conditions if not
explicitly added. We should rather similate the generation instead.

This commit also adds a random test to check if the settings
of the all field mapper are correctly applied.

Closes #4579
Closes #4581
  • Loading branch information
@s1monw
s1monw committed Jan 2, 2014
1 parent c842519 commit 9a2af1b
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 12 deletions.
31 changes: 20 additions & 11 deletions src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View file
Expand Up @@ -28,6 +28,7 @@
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.all.AllField;
import org.elasticsearch.common.lucene.all.AllTermQuery;
Expand Down Expand Up @@ -250,16 +251,26 @@ protected String contentType() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// if all are defaults, no need to write it at all
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);

if (!includeDefaults && enabled == Defaults.ENABLED && fieldType.stored() == Defaults.FIELD_TYPE.stored() &&
fieldType.storeTermVectors() == Defaults.FIELD_TYPE.storeTermVectors() &&
indexAnalyzer == null && searchAnalyzer == null
&& fieldType.omitNorms() == Defaults.FIELD_TYPE.omitNorms()) {
return builder;
if (!includeDefaults) {
// simulate the generation to make sure we don't add unnecessary content if all is default
// if all are defaults, no need to write it at all - generating is twice is ok though
BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
XContentBuilder b = new XContentBuilder(builder.contentType().xContent(), bytesStreamOutput);
long pos = bytesStreamOutput.position();
innerToXContent(b, false);
b.flush();
if (pos == bytesStreamOutput.position()) {
return builder;
}
}
builder.startObject(CONTENT_TYPE);
innerToXContent(builder, includeDefaults);
builder.endObject();
return builder;
}

private void innerToXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
if (includeDefaults || enabled != Defaults.ENABLED) {
builder.field("enabled", enabled);
}
Expand All @@ -270,7 +281,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field("store", fieldType.stored());
}
if (includeDefaults || fieldType.storeTermVectors() != Defaults.FIELD_TYPE.storeTermVectors()) {
builder.field("store_term_vector", fieldType.storeTermVectors());
builder.field("store_term_vectors", fieldType.storeTermVectors());
}
if (includeDefaults || fieldType.storeTermVectorOffsets() != Defaults.FIELD_TYPE.storeTermVectorOffsets()) {
builder.field("store_term_vector_offsets", fieldType.storeTermVectorOffsets());
Expand Down Expand Up @@ -326,11 +337,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
} else if (includeDefaults) {
builder.field("fielddata", (Map) fieldDataType.getSettings().getAsMap());
}

builder.endObject();
return builder;
}


@Override
public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
// do nothing here, no merging, but also no exception
Expand Down
130 changes: 129 additions & 1 deletion src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View file
Expand Up @@ -21,22 +21,33 @@

import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.lucene.all.AllEntries;
import org.elasticsearch.common.lucene.all.AllField;
import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.lucene.all.AllTokenStream;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperTestUtils;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.Matchers;
import org.junit.Test;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.hamcrest.Matchers.equalTo;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.*;

/**
*
Expand Down Expand Up @@ -152,4 +163,121 @@ public void testSimpleAllMappersWithReparseWithStore() throws Exception {
assertThat(text, equalTo(allEntries.buildText()));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}

@Test
public void testRandom() throws Exception {
boolean omitNorms = false;
boolean stored = false;
boolean enabled = true;
boolean autoBoost = false;
boolean tv_stored = false;
boolean tv_payloads = false;
boolean tv_offsets = false;
boolean tv_positions = false;
String similarity = null;
boolean fieldData = false;
XContentBuilder mappingBuilder = jsonBuilder();
mappingBuilder.startObject().startObject("test");
List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<Tuple<String, Boolean>>();
boolean allDefault = true;
if (frequently()) {
allDefault = false;
mappingBuilder.startObject("_all");
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("omit_norms", omitNorms = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store", stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store_term_vectors", tv_stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("enabled", enabled = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("auto_boost", autoBoost = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store_term_vector_offsets", tv_offsets = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store_term_vector_positions", tv_positions = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store_term_vector_payloads", tv_payloads = randomBoolean()));
}
Collections.shuffle(booleanOptionList, getRandom());
for (Tuple<String, Boolean> option : booleanOptionList) {
mappingBuilder.field(option.v1(), option.v2().booleanValue());
}
tv_stored |= tv_positions || tv_payloads || tv_offsets;
if (randomBoolean()) {
mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
}
if (randomBoolean()) {
fieldData = true;
mappingBuilder.startObject("fielddata");
mappingBuilder.field("foo", "bar");
mappingBuilder.endObject();
}
mappingBuilder.endObject();
}

String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
logger.info(mapping);
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);

byte[] json = jsonBuilder().startObject().startObject("test")
.field("foo", "bar")
.field("_id", 1)
.field("foobar", "foobar")
.endObject().endObject().bytes().array();
Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
if (enabled) {
assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
assertThat(field.fieldType().stored(), equalTo(stored));
assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
AllEntries allEntries = ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("foobar"), equalTo(true));
assertThat(allEntries.fields().contains("foo"), equalTo(true));
if (!stored) {
assertThat(field.stringValue(), nullValue());
}
String text = stored ? field.stringValue() : "bar foobar";
assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
} else {
assertThat(field, nullValue());
}

Term term = new Term("foo", "bar");
Query query = builtDocMapper.allFieldMapper().queryStringTermQuery(term);
if (autoBoost) {
assertThat(query, equalTo((Query)new AllTermQuery(term)));
} else {
assertThat(query, equalTo((Query)new TermQuery(term)));
}
if (similarity == null || similarity.equals("TF/IDF")) {
assertThat(builtDocMapper.allFieldMapper().similarity(), nullValue());
} else {
assertThat(similarity, equalTo(builtDocMapper.allFieldMapper().similarity().name()));
}
assertThat(builtMapping.contains("fielddata"), is(fieldData));
if (allDefault) {
BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
XContentBuilder xContentBuilder = builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
xContentBuilder.flush();
assertThat(bytesStreamOutput.size(), equalTo(0));
}

}
}

0 comments on commit 9a2af1b

Please sign in to comment.